2024-11-24 13:46:55,058 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-11-24 13:46:55,071 main DEBUG Took 0.011140 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 13:46:55,071 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 13:46:55,072 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 13:46:55,073 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 13:46:55,074 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:46:55,082 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 13:46:55,095 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:46:55,097 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:46:55,097 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:46:55,098 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:46:55,099 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:46:55,099 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:46:55,100 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:46:55,100 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:46:55,101 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:46:55,101 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:46:55,102 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:46:55,102 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:46:55,103 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:46:55,103 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:46:55,104 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:46:55,104 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:46:55,104 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:46:55,105 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:46:55,105 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:46:55,106 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:46:55,106 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:46:55,106 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:46:55,107 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:46:55,107 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:46:55,108 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:46:55,108 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 13:46:55,109 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:46:55,111 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 13:46:55,113 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 13:46:55,113 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 13:46:55,114 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 13:46:55,115 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 13:46:55,123 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 13:46:55,126 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 13:46:55,128 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 13:46:55,128 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 13:46:55,129 main DEBUG createAppenders(={Console}) 2024-11-24 13:46:55,130 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b initialized 2024-11-24 13:46:55,130 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-11-24 13:46:55,130 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b OK. 2024-11-24 13:46:55,131 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 13:46:55,131 main DEBUG OutputStream closed 2024-11-24 13:46:55,132 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 13:46:55,132 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 13:46:55,132 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@37d4349f OK 2024-11-24 13:46:55,203 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 13:46:55,205 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 13:46:55,206 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 13:46:55,208 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 13:46:55,208 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 13:46:55,209 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 13:46:55,209 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 13:46:55,209 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 13:46:55,210 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 13:46:55,210 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 13:46:55,210 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 13:46:55,211 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 13:46:55,211 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 13:46:55,211 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 13:46:55,211 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 13:46:55,212 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 13:46:55,212 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 13:46:55,213 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 13:46:55,215 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 13:46:55,215 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4331d187) with optional ClassLoader: null 2024-11-24 13:46:55,216 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 13:46:55,216 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4331d187] started OK. 2024-11-24T13:46:55,228 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync timeout: 13 mins 2024-11-24 13:46:55,231 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 13:46:55,231 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T13:46:55,280 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.TestFailedAppendAndSync#testLockupAroundBadAssignSync Thread=11, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=153, ProcessCount=11, AvailableMemoryMB=9171 2024-11-24T13:46:55,521 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf 2024-11-24T13:46:56,026 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T13:46:56,121 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T13:46:56,174 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=wal, suffix=, logDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync, archiveDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/oldWALs, maxLogs=55 2024-11-24T13:46:56,221 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456016211 2024-11-24T13:46:56,285 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016211 2024-11-24T13:46:56,351 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:56,360 INFO [Time-limited test {}] regionserver.ChunkCreator(472): data poolSizePercentage is less than 0. So not using pool 2024-11-24T13:46:56,361 INFO [Time-limited test {}] regionserver.ChunkCreator(472): index poolSizePercentage is less than 0. So not using pool 2024-11-24T13:46:56,398 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => f95a3003458fa47569d2022d5cc40612, NAME => 'testLockupAroundBadAssignSync,,1732456016378.f95a3003458fa47569d2022d5cc40612.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testLockupAroundBadAssignSync', {TABLE_ATTRIBUTES => {DURABILITY => 'SYNC_WAL', READONLY => 'false'}}, {NAME => 'MyCF', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)', METADATA => {'IN_MEMORY_COMPACTION' => 'NONE'}}, regionDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf 2024-11-24T13:46:56,439 INFO [Time-limited test {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T13:46:56,444 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testLockupAroundBadAssignSync,,1732456016378.f95a3003458fa47569d2022d5cc40612.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:46:56,540 INFO [StoreOpener-f95a3003458fa47569d2022d5cc40612-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family MyCF of region f95a3003458fa47569d2022d5cc40612 2024-11-24T13:46:56,577 INFO [StoreOpener-f95a3003458fa47569d2022d5cc40612-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f95a3003458fa47569d2022d5cc40612 columnFamilyName MyCF 2024-11-24T13:46:56,584 DEBUG [StoreOpener-f95a3003458fa47569d2022d5cc40612-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:46:56,591 INFO [StoreOpener-f95a3003458fa47569d2022d5cc40612-1 {}] regionserver.HStore(327): Store=f95a3003458fa47569d2022d5cc40612/MyCF, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:46:56,594 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f95a3003458fa47569d2022d5cc40612 2024-11-24T13:46:56,601 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/data/default/testLockupAroundBadAssignSync/f95a3003458fa47569d2022d5cc40612 2024-11-24T13:46:56,602 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/data/default/testLockupAroundBadAssignSync/f95a3003458fa47569d2022d5cc40612 2024-11-24T13:46:56,604 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f95a3003458fa47569d2022d5cc40612 2024-11-24T13:46:56,605 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f95a3003458fa47569d2022d5cc40612 2024-11-24T13:46:56,612 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f95a3003458fa47569d2022d5cc40612 2024-11-24T13:46:56,622 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/data/default/testLockupAroundBadAssignSync/f95a3003458fa47569d2022d5cc40612/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T13:46:56,623 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f95a3003458fa47569d2022d5cc40612; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63142933, jitterRate=-0.05909697711467743}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T13:46:56,629 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T13:46:56,643 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f95a3003458fa47569d2022d5cc40612: Writing region info on filesystem at 1732456016500Initializing all the Stores at 1732456016501 (+1 ms)Instantiating store for column family {NAME => 'MyCF', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)', METADATA => {'IN_MEMORY_COMPACTION' => 'NONE'}} at 1732456016502 (+1 ms)Cleaning up temporary data from old regions at 1732456016605 (+103 ms)Region opened successfully at 1732456016641 (+36 ms) 2024-11-24T13:46:56,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456016211) roll requested 2024-11-24T13:46:56,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456016686 2024-11-24T13:46:56,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016211 with entries=1, filesize=249 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016686 2024-11-24T13:46:56,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:56,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456016686) roll requested 2024-11-24T13:46:56,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456016696 2024-11-24T13:46:56,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016696 2024-11-24T13:46:56,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:56,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456016696) roll requested 2024-11-24T13:46:56,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456016706 2024-11-24T13:46:56,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016706 2024-11-24T13:46:56,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:56,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456016706) roll requested 2024-11-24T13:46:56,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456016716 2024-11-24T13:46:56,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016706 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016716 2024-11-24T13:46:56,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:56,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456016716) roll requested 2024-11-24T13:46:56,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456016727 2024-11-24T13:46:56,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016716 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016727 2024-11-24T13:46:56,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:56,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456016727) roll requested 2024-11-24T13:46:56,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456016737 2024-11-24T13:46:56,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016737 2024-11-24T13:46:56,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:56,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456016737) roll requested 2024-11-24T13:46:56,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456016750 2024-11-24T13:46:56,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016750 2024-11-24T13:46:56,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:56,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456016750) roll requested 2024-11-24T13:46:56,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456016760 2024-11-24T13:46:56,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016760 2024-11-24T13:46:56,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:56,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456016760) roll requested 2024-11-24T13:46:56,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456016773 2024-11-24T13:46:56,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016773 2024-11-24T13:46:56,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:56,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456016773) roll requested 2024-11-24T13:46:56,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456016783 2024-11-24T13:46:56,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016783 2024-11-24T13:46:56,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:56,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456016783) roll requested 2024-11-24T13:46:56,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456016796 2024-11-24T13:46:56,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016796 2024-11-24T13:46:56,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:56,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456016796) roll requested 2024-11-24T13:46:56,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456016804 2024-11-24T13:46:56,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016804 2024-11-24T13:46:56,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:56,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456016804) roll requested 2024-11-24T13:46:56,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456016812 2024-11-24T13:46:56,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016812 2024-11-24T13:46:56,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:56,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456016812) roll requested 2024-11-24T13:46:56,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456016822 2024-11-24T13:46:56,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016822 2024-11-24T13:46:56,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:56,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456016822) roll requested 2024-11-24T13:46:56,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456016832 2024-11-24T13:46:56,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016832 2024-11-24T13:46:56,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:56,846 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456016832) roll requested 2024-11-24T13:46:56,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456016847 2024-11-24T13:46:56,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016847 2024-11-24T13:46:56,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:56,858 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456016847) roll requested 2024-11-24T13:46:56,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456016859 2024-11-24T13:46:56,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016859 2024-11-24T13:46:56,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:56,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456016859) roll requested 2024-11-24T13:46:56,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456016870 2024-11-24T13:46:56,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016859 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016870 2024-11-24T13:46:56,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:56,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456016870) roll requested 2024-11-24T13:46:56,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456016881 2024-11-24T13:46:56,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016881 2024-11-24T13:46:56,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:56,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456016881) roll requested 2024-11-24T13:46:56,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456016889 2024-11-24T13:46:56,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016889 2024-11-24T13:46:56,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:56,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456016889) roll requested 2024-11-24T13:46:56,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456016899 2024-11-24T13:46:56,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016899 2024-11-24T13:46:56,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:56,913 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456016899) roll requested 2024-11-24T13:46:56,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456016914 2024-11-24T13:46:56,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016899 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016914 2024-11-24T13:46:56,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:56,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456016914) roll requested 2024-11-24T13:46:56,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456016926 2024-11-24T13:46:56,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016926 2024-11-24T13:46:56,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:56,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456016926) roll requested 2024-11-24T13:46:56,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456016936 2024-11-24T13:46:56,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016936 2024-11-24T13:46:56,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:56,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456016936) roll requested 2024-11-24T13:46:56,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456016944 2024-11-24T13:46:56,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016944 2024-11-24T13:46:56,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:56,950 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456016944) roll requested 2024-11-24T13:46:56,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456016951 2024-11-24T13:46:56,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016951 2024-11-24T13:46:56,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:56,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456016951) roll requested 2024-11-24T13:46:56,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456016958 2024-11-24T13:46:56,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016951 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016958 2024-11-24T13:46:56,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:56,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456016958) roll requested 2024-11-24T13:46:56,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456016967 2024-11-24T13:46:56,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016967 2024-11-24T13:46:56,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:56,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456016967) roll requested 2024-11-24T13:46:56,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456016977 2024-11-24T13:46:56,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016977 2024-11-24T13:46:56,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:56,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456016977) roll requested 2024-11-24T13:46:56,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456016988 2024-11-24T13:46:56,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:56,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016988 2024-11-24T13:46:56,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:56,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:56,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456016988) roll requested 2024-11-24T13:46:56,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456016996 2024-11-24T13:46:57,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016996 2024-11-24T13:46:57,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456016996) roll requested 2024-11-24T13:46:57,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017004 2024-11-24T13:46:57,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456016996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017004 2024-11-24T13:46:57,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017004) roll requested 2024-11-24T13:46:57,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017013 2024-11-24T13:46:57,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017013 2024-11-24T13:46:57,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017013) roll requested 2024-11-24T13:46:57,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017021 2024-11-24T13:46:57,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017021 2024-11-24T13:46:57,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,029 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017021) roll requested 2024-11-24T13:46:57,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017029 2024-11-24T13:46:57,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017029 2024-11-24T13:46:57,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017029) roll requested 2024-11-24T13:46:57,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017037 2024-11-24T13:46:57,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017037 2024-11-24T13:46:57,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,045 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017037) roll requested 2024-11-24T13:46:57,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017046 2024-11-24T13:46:57,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017046 2024-11-24T13:46:57,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017046) roll requested 2024-11-24T13:46:57,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017055 2024-11-24T13:46:57,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017046 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017055 2024-11-24T13:46:57,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017055) roll requested 2024-11-24T13:46:57,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017065 2024-11-24T13:46:57,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017065 2024-11-24T13:46:57,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017065) roll requested 2024-11-24T13:46:57,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017075 2024-11-24T13:46:57,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017075 2024-11-24T13:46:57,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017075) roll requested 2024-11-24T13:46:57,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017083 2024-11-24T13:46:57,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017083 2024-11-24T13:46:57,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017083) roll requested 2024-11-24T13:46:57,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017089 2024-11-24T13:46:57,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017089 2024-11-24T13:46:57,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,101 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017089) roll requested 2024-11-24T13:46:57,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017102 2024-11-24T13:46:57,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017102 2024-11-24T13:46:57,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017102) roll requested 2024-11-24T13:46:57,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017113 2024-11-24T13:46:57,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017102 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017113 2024-11-24T13:46:57,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017113) roll requested 2024-11-24T13:46:57,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017120 2024-11-24T13:46:57,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017120 2024-11-24T13:46:57,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017120) roll requested 2024-11-24T13:46:57,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017128 2024-11-24T13:46:57,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017128 2024-11-24T13:46:57,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017128) roll requested 2024-11-24T13:46:57,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017135 2024-11-24T13:46:57,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017135 2024-11-24T13:46:57,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017135) roll requested 2024-11-24T13:46:57,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017143 2024-11-24T13:46:57,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017143 2024-11-24T13:46:57,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017143) roll requested 2024-11-24T13:46:57,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017152 2024-11-24T13:46:57,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017152 2024-11-24T13:46:57,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,166 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017152) roll requested 2024-11-24T13:46:57,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017167 2024-11-24T13:46:57,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017167 2024-11-24T13:46:57,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017167) roll requested 2024-11-24T13:46:57,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017179 2024-11-24T13:46:57,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017179 2024-11-24T13:46:57,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017179) roll requested 2024-11-24T13:46:57,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017189 2024-11-24T13:46:57,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017189 2024-11-24T13:46:57,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017189) roll requested 2024-11-24T13:46:57,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017203 2024-11-24T13:46:57,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017203 2024-11-24T13:46:57,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017203) roll requested 2024-11-24T13:46:57,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017213 2024-11-24T13:46:57,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017203 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017213 2024-11-24T13:46:57,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017213) roll requested 2024-11-24T13:46:57,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017220 2024-11-24T13:46:57,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017220 2024-11-24T13:46:57,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017220) roll requested 2024-11-24T13:46:57,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017227 2024-11-24T13:46:57,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017227 2024-11-24T13:46:57,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,235 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017227) roll requested 2024-11-24T13:46:57,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017235 2024-11-24T13:46:57,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017235 2024-11-24T13:46:57,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,243 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017235) roll requested 2024-11-24T13:46:57,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017244 2024-11-24T13:46:57,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017235 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017244 2024-11-24T13:46:57,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017244) roll requested 2024-11-24T13:46:57,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017252 2024-11-24T13:46:57,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017252 2024-11-24T13:46:57,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017252) roll requested 2024-11-24T13:46:57,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017260 2024-11-24T13:46:57,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017260 2024-11-24T13:46:57,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017260) roll requested 2024-11-24T13:46:57,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017274 2024-11-24T13:46:57,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017274 2024-11-24T13:46:57,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,282 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017274) roll requested 2024-11-24T13:46:57,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017282 2024-11-24T13:46:57,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017282 2024-11-24T13:46:57,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017282) roll requested 2024-11-24T13:46:57,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017290 2024-11-24T13:46:57,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017290 2024-11-24T13:46:57,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017290) roll requested 2024-11-24T13:46:57,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017298 2024-11-24T13:46:57,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017290 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017298 2024-11-24T13:46:57,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,309 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017298) roll requested 2024-11-24T13:46:57,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017310 2024-11-24T13:46:57,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017310 2024-11-24T13:46:57,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017310) roll requested 2024-11-24T13:46:57,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017319 2024-11-24T13:46:57,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017319 2024-11-24T13:46:57,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017319) roll requested 2024-11-24T13:46:57,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017327 2024-11-24T13:46:57,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017327 2024-11-24T13:46:57,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,333 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017327) roll requested 2024-11-24T13:46:57,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017334 2024-11-24T13:46:57,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017334 2024-11-24T13:46:57,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017334) roll requested 2024-11-24T13:46:57,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017341 2024-11-24T13:46:57,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017341 2024-11-24T13:46:57,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017341) roll requested 2024-11-24T13:46:57,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017348 2024-11-24T13:46:57,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017348 2024-11-24T13:46:57,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017348) roll requested 2024-11-24T13:46:57,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017355 2024-11-24T13:46:57,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017355 2024-11-24T13:46:57,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017355) roll requested 2024-11-24T13:46:57,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017362 2024-11-24T13:46:57,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017362 2024-11-24T13:46:57,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017355 is not closed yet, will try archiving it next time 2024-11-24T13:46:57,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017362) roll requested 2024-11-24T13:46:57,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017370 2024-11-24T13:46:57,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017370 2024-11-24T13:46:57,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017370) roll requested 2024-11-24T13:46:57,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017379 2024-11-24T13:46:57,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017379 2024-11-24T13:46:57,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017379) roll requested 2024-11-24T13:46:57,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017388 2024-11-24T13:46:57,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017388 2024-11-24T13:46:57,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017388) roll requested 2024-11-24T13:46:57,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017398 2024-11-24T13:46:57,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017398 2024-11-24T13:46:57,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017398) roll requested 2024-11-24T13:46:57,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017406 2024-11-24T13:46:57,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017398 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017406 2024-11-24T13:46:57,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017406) roll requested 2024-11-24T13:46:57,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017416 2024-11-24T13:46:57,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017416 2024-11-24T13:46:57,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017416) roll requested 2024-11-24T13:46:57,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017427 2024-11-24T13:46:57,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017427 2024-11-24T13:46:57,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017427) roll requested 2024-11-24T13:46:57,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017436 2024-11-24T13:46:57,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017436 2024-11-24T13:46:57,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017436) roll requested 2024-11-24T13:46:57,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017446 2024-11-24T13:46:57,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017446 2024-11-24T13:46:57,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017446) roll requested 2024-11-24T13:46:57,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017453 2024-11-24T13:46:57,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017453 2024-11-24T13:46:57,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,462 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017453) roll requested 2024-11-24T13:46:57,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017463 2024-11-24T13:46:57,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017463 2024-11-24T13:46:57,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017463) roll requested 2024-11-24T13:46:57,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017473 2024-11-24T13:46:57,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017473 2024-11-24T13:46:57,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017473) roll requested 2024-11-24T13:46:57,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017481 2024-11-24T13:46:57,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017481 2024-11-24T13:46:57,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017481) roll requested 2024-11-24T13:46:57,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017488 2024-11-24T13:46:57,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017488 2024-11-24T13:46:57,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017488) roll requested 2024-11-24T13:46:57,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017496 2024-11-24T13:46:57,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017496 2024-11-24T13:46:57,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017496) roll requested 2024-11-24T13:46:57,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017505 2024-11-24T13:46:57,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017505 2024-11-24T13:46:57,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017505) roll requested 2024-11-24T13:46:57,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017517 2024-11-24T13:46:57,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017517 2024-11-24T13:46:57,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,525 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017517) roll requested 2024-11-24T13:46:57,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017526 2024-11-24T13:46:57,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017526 2024-11-24T13:46:57,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017526) roll requested 2024-11-24T13:46:57,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017536 2024-11-24T13:46:57,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017536 2024-11-24T13:46:57,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,545 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017536) roll requested 2024-11-24T13:46:57,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017546 2024-11-24T13:46:57,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017546 2024-11-24T13:46:57,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017546) roll requested 2024-11-24T13:46:57,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017556 2024-11-24T13:46:57,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017556 2024-11-24T13:46:57,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017556) roll requested 2024-11-24T13:46:57,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017562 2024-11-24T13:46:57,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017562 2024-11-24T13:46:57,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017562) roll requested 2024-11-24T13:46:57,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017569 2024-11-24T13:46:57,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017562 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017569 2024-11-24T13:46:57,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017569) roll requested 2024-11-24T13:46:57,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017578 2024-11-24T13:46:57,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017578 2024-11-24T13:46:57,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,586 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017578) roll requested 2024-11-24T13:46:57,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017587 2024-11-24T13:46:57,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017587 2024-11-24T13:46:57,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017587) roll requested 2024-11-24T13:46:57,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017597 2024-11-24T13:46:57,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017597 2024-11-24T13:46:57,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017597) roll requested 2024-11-24T13:46:57,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017604 2024-11-24T13:46:57,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017604 2024-11-24T13:46:57,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017604) roll requested 2024-11-24T13:46:57,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017614 2024-11-24T13:46:57,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017614 2024-11-24T13:46:57,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,622 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017614) roll requested 2024-11-24T13:46:57,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017622 2024-11-24T13:46:57,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017622 2024-11-24T13:46:57,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,630 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017622) roll requested 2024-11-24T13:46:57,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017631 2024-11-24T13:46:57,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017622 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017631 2024-11-24T13:46:57,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017631) roll requested 2024-11-24T13:46:57,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017641 2024-11-24T13:46:57,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017641 2024-11-24T13:46:57,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017641) roll requested 2024-11-24T13:46:57,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017649 2024-11-24T13:46:57,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017649 2024-11-24T13:46:57,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017649) roll requested 2024-11-24T13:46:57,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017659 2024-11-24T13:46:57,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017659 2024-11-24T13:46:57,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017659) roll requested 2024-11-24T13:46:57,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017671 2024-11-24T13:46:57,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017671 2024-11-24T13:46:57,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017671) roll requested 2024-11-24T13:46:57,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017681 2024-11-24T13:46:57,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017671 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017681 2024-11-24T13:46:57,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017681) roll requested 2024-11-24T13:46:57,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017690 2024-11-24T13:46:57,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017690 2024-11-24T13:46:57,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017690) roll requested 2024-11-24T13:46:57,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017698 2024-11-24T13:46:57,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017698 2024-11-24T13:46:57,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017698) roll requested 2024-11-24T13:46:57,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017705 2024-11-24T13:46:57,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017705 2024-11-24T13:46:57,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017705) roll requested 2024-11-24T13:46:57,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017711 2024-11-24T13:46:57,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017711 2024-11-24T13:46:57,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017711) roll requested 2024-11-24T13:46:57,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017720 2024-11-24T13:46:57,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017720 2024-11-24T13:46:57,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017720) roll requested 2024-11-24T13:46:57,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017729 2024-11-24T13:46:57,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017729 2024-11-24T13:46:57,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017729) roll requested 2024-11-24T13:46:57,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017739 2024-11-24T13:46:57,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017739 2024-11-24T13:46:57,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017739) roll requested 2024-11-24T13:46:57,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017751 2024-11-24T13:46:57,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017751 2024-11-24T13:46:57,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017751) roll requested 2024-11-24T13:46:57,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017760 2024-11-24T13:46:57,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017760 2024-11-24T13:46:57,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,766 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017760) roll requested 2024-11-24T13:46:57,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017767 2024-11-24T13:46:57,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017767 2024-11-24T13:46:57,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017767) roll requested 2024-11-24T13:46:57,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017776 2024-11-24T13:46:57,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017776 2024-11-24T13:46:57,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017776) roll requested 2024-11-24T13:46:57,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017786 2024-11-24T13:46:57,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017786 2024-11-24T13:46:57,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017786) roll requested 2024-11-24T13:46:57,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017795 2024-11-24T13:46:57,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017795 2024-11-24T13:46:57,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017795) roll requested 2024-11-24T13:46:57,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017803 2024-11-24T13:46:57,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017803 2024-11-24T13:46:57,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,810 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017803) roll requested 2024-11-24T13:46:57,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017811 2024-11-24T13:46:57,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017811 2024-11-24T13:46:57,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017811) roll requested 2024-11-24T13:46:57,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017819 2024-11-24T13:46:57,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017819 2024-11-24T13:46:57,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017819) roll requested 2024-11-24T13:46:57,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017829 2024-11-24T13:46:57,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017819 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017829 2024-11-24T13:46:57,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017829) roll requested 2024-11-24T13:46:57,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017840 2024-11-24T13:46:57,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017840 2024-11-24T13:46:57,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017840) roll requested 2024-11-24T13:46:57,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017851 2024-11-24T13:46:57,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017851 2024-11-24T13:46:57,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,858 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017851) roll requested 2024-11-24T13:46:57,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017858 2024-11-24T13:46:57,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017858 2024-11-24T13:46:57,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,866 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017858) roll requested 2024-11-24T13:46:57,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017867 2024-11-24T13:46:57,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017867 2024-11-24T13:46:57,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,874 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017867) roll requested 2024-11-24T13:46:57,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017875 2024-11-24T13:46:57,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017875 2024-11-24T13:46:57,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017875) roll requested 2024-11-24T13:46:57,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017883 2024-11-24T13:46:57,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017875 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017883 2024-11-24T13:46:57,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017883) roll requested 2024-11-24T13:46:57,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017895 2024-11-24T13:46:57,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017895 2024-11-24T13:46:57,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017895) roll requested 2024-11-24T13:46:57,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017905 2024-11-24T13:46:57,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017905 2024-11-24T13:46:57,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017905) roll requested 2024-11-24T13:46:57,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017911 2024-11-24T13:46:57,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017911 2024-11-24T13:46:57,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017911) roll requested 2024-11-24T13:46:57,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017917 2024-11-24T13:46:57,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017917 2024-11-24T13:46:57,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,923 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017917) roll requested 2024-11-24T13:46:57,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017924 2024-11-24T13:46:57,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017924 2024-11-24T13:46:57,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017924) roll requested 2024-11-24T13:46:57,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017937 2024-11-24T13:46:57,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017937 2024-11-24T13:46:57,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017937) roll requested 2024-11-24T13:46:57,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017948 2024-11-24T13:46:57,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017948 2024-11-24T13:46:57,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017948) roll requested 2024-11-24T13:46:57,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017956 2024-11-24T13:46:57,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017948 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017956 2024-11-24T13:46:57,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,963 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017956) roll requested 2024-11-24T13:46:57,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017963 2024-11-24T13:46:57,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017963 2024-11-24T13:46:57,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017963) roll requested 2024-11-24T13:46:57,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017972 2024-11-24T13:46:57,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017963 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017972 2024-11-24T13:46:57,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017972) roll requested 2024-11-24T13:46:57,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017981 2024-11-24T13:46:57,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:57,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017981 2024-11-24T13:46:57,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:57,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:57,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017981) roll requested 2024-11-24T13:46:57,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456017997 2024-11-24T13:46:58,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017997 2024-11-24T13:46:58,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456017997) roll requested 2024-11-24T13:46:58,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018007 2024-11-24T13:46:58,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456017997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018007 2024-11-24T13:46:58,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018007) roll requested 2024-11-24T13:46:58,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018016 2024-11-24T13:46:58,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018016 2024-11-24T13:46:58,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018016) roll requested 2024-11-24T13:46:58,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018023 2024-11-24T13:46:58,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018023 2024-11-24T13:46:58,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018023) roll requested 2024-11-24T13:46:58,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018032 2024-11-24T13:46:58,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018032 2024-11-24T13:46:58,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,038 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018032) roll requested 2024-11-24T13:46:58,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018038 2024-11-24T13:46:58,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018038 2024-11-24T13:46:58,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,045 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018038) roll requested 2024-11-24T13:46:58,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018045 2024-11-24T13:46:58,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018038 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018045 2024-11-24T13:46:58,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018045) roll requested 2024-11-24T13:46:58,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018052 2024-11-24T13:46:58,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018052 2024-11-24T13:46:58,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,058 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018052) roll requested 2024-11-24T13:46:58,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018058 2024-11-24T13:46:58,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018058 2024-11-24T13:46:58,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018058) roll requested 2024-11-24T13:46:58,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018070 2024-11-24T13:46:58,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018058 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018070 2024-11-24T13:46:58,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018070) roll requested 2024-11-24T13:46:58,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018079 2024-11-24T13:46:58,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018079 2024-11-24T13:46:58,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018079) roll requested 2024-11-24T13:46:58,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018086 2024-11-24T13:46:58,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018086 2024-11-24T13:46:58,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018086) roll requested 2024-11-24T13:46:58,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018093 2024-11-24T13:46:58,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018093 2024-11-24T13:46:58,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018093) roll requested 2024-11-24T13:46:58,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018099 2024-11-24T13:46:58,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018099 2024-11-24T13:46:58,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018099) roll requested 2024-11-24T13:46:58,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018107 2024-11-24T13:46:58,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018107 2024-11-24T13:46:58,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018107) roll requested 2024-11-24T13:46:58,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018115 2024-11-24T13:46:58,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018115 2024-11-24T13:46:58,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018115) roll requested 2024-11-24T13:46:58,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018122 2024-11-24T13:46:58,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018115 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018122 2024-11-24T13:46:58,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018122) roll requested 2024-11-24T13:46:58,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018129 2024-11-24T13:46:58,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018129 2024-11-24T13:46:58,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018129) roll requested 2024-11-24T13:46:58,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018135 2024-11-24T13:46:58,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018135 2024-11-24T13:46:58,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018135) roll requested 2024-11-24T13:46:58,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018142 2024-11-24T13:46:58,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018142 2024-11-24T13:46:58,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018142) roll requested 2024-11-24T13:46:58,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018151 2024-11-24T13:46:58,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018151 2024-11-24T13:46:58,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018151) roll requested 2024-11-24T13:46:58,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018159 2024-11-24T13:46:58,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018159 2024-11-24T13:46:58,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,167 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018159) roll requested 2024-11-24T13:46:58,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018167 2024-11-24T13:46:58,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018167 2024-11-24T13:46:58,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,174 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018167) roll requested 2024-11-24T13:46:58,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018174 2024-11-24T13:46:58,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018174 2024-11-24T13:46:58,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,181 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018174) roll requested 2024-11-24T13:46:58,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018181 2024-11-24T13:46:58,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018174 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018181 2024-11-24T13:46:58,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018181) roll requested 2024-11-24T13:46:58,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018188 2024-11-24T13:46:58,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018188 2024-11-24T13:46:58,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018188) roll requested 2024-11-24T13:46:58,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018199 2024-11-24T13:46:58,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018199 2024-11-24T13:46:58,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018199) roll requested 2024-11-24T13:46:58,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018205 2024-11-24T13:46:58,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018199 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018205 2024-11-24T13:46:58,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018205) roll requested 2024-11-24T13:46:58,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018215 2024-11-24T13:46:58,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018215 2024-11-24T13:46:58,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018215) roll requested 2024-11-24T13:46:58,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018221 2024-11-24T13:46:58,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018221 2024-11-24T13:46:58,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018221) roll requested 2024-11-24T13:46:58,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018228 2024-11-24T13:46:58,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018228 2024-11-24T13:46:58,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,235 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018228) roll requested 2024-11-24T13:46:58,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018235 2024-11-24T13:46:58,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018235 2024-11-24T13:46:58,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018235) roll requested 2024-11-24T13:46:58,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018243 2024-11-24T13:46:58,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018235 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018243 2024-11-24T13:46:58,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018243) roll requested 2024-11-24T13:46:58,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018251 2024-11-24T13:46:58,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018243 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018251 2024-11-24T13:46:58,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,258 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018251) roll requested 2024-11-24T13:46:58,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018258 2024-11-24T13:46:58,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018258 2024-11-24T13:46:58,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,266 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018258) roll requested 2024-11-24T13:46:58,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018266 2024-11-24T13:46:58,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018258 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018266 2024-11-24T13:46:58,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018266) roll requested 2024-11-24T13:46:58,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018273 2024-11-24T13:46:58,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018273 2024-11-24T13:46:58,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018273) roll requested 2024-11-24T13:46:58,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018281 2024-11-24T13:46:58,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018281 2024-11-24T13:46:58,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018281) roll requested 2024-11-24T13:46:58,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018287 2024-11-24T13:46:58,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018287 2024-11-24T13:46:58,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018287) roll requested 2024-11-24T13:46:58,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018296 2024-11-24T13:46:58,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018296 2024-11-24T13:46:58,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,303 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018296) roll requested 2024-11-24T13:46:58,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018303 2024-11-24T13:46:58,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018296 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018303 2024-11-24T13:46:58,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018303) roll requested 2024-11-24T13:46:58,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018310 2024-11-24T13:46:58,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018303 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018310 2024-11-24T13:46:58,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018310) roll requested 2024-11-24T13:46:58,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018320 2024-11-24T13:46:58,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018320 2024-11-24T13:46:58,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018320) roll requested 2024-11-24T13:46:58,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018327 2024-11-24T13:46:58,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018327 2024-11-24T13:46:58,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018327) roll requested 2024-11-24T13:46:58,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018334 2024-11-24T13:46:58,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018334 2024-11-24T13:46:58,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018334) roll requested 2024-11-24T13:46:58,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018340 2024-11-24T13:46:58,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018340 2024-11-24T13:46:58,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018340) roll requested 2024-11-24T13:46:58,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018347 2024-11-24T13:46:58,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018347 2024-11-24T13:46:58,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018347) roll requested 2024-11-24T13:46:58,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018354 2024-11-24T13:46:58,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018354 2024-11-24T13:46:58,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018354) roll requested 2024-11-24T13:46:58,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018363 2024-11-24T13:46:58,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018363 2024-11-24T13:46:58,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018363) roll requested 2024-11-24T13:46:58,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018371 2024-11-24T13:46:58,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018371 2024-11-24T13:46:58,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018371) roll requested 2024-11-24T13:46:58,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018379 2024-11-24T13:46:58,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018379 2024-11-24T13:46:58,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018379) roll requested 2024-11-24T13:46:58,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018389 2024-11-24T13:46:58,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018389 2024-11-24T13:46:58,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018389) roll requested 2024-11-24T13:46:58,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018397 2024-11-24T13:46:58,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018397 2024-11-24T13:46:58,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018397) roll requested 2024-11-24T13:46:58,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018405 2024-11-24T13:46:58,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018405 2024-11-24T13:46:58,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018405) roll requested 2024-11-24T13:46:58,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018412 2024-11-24T13:46:58,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018412 2024-11-24T13:46:58,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,423 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018412) roll requested 2024-11-24T13:46:58,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018423 2024-11-24T13:46:58,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018423 2024-11-24T13:46:58,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018423) roll requested 2024-11-24T13:46:58,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018432 2024-11-24T13:46:58,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018423 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018432 2024-11-24T13:46:58,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,442 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018432) roll requested 2024-11-24T13:46:58,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018443 2024-11-24T13:46:58,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018443 2024-11-24T13:46:58,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018443) roll requested 2024-11-24T13:46:58,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018451 2024-11-24T13:46:58,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018443 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018451 2024-11-24T13:46:58,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018451) roll requested 2024-11-24T13:46:58,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018457 2024-11-24T13:46:58,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018457 2024-11-24T13:46:58,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018457) roll requested 2024-11-24T13:46:58,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018467 2024-11-24T13:46:58,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018467 2024-11-24T13:46:58,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018467) roll requested 2024-11-24T13:46:58,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018475 2024-11-24T13:46:58,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018475 2024-11-24T13:46:58,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018475) roll requested 2024-11-24T13:46:58,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018482 2024-11-24T13:46:58,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018482 2024-11-24T13:46:58,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018482) roll requested 2024-11-24T13:46:58,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018489 2024-11-24T13:46:58,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018482 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018489 2024-11-24T13:46:58,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018489) roll requested 2024-11-24T13:46:58,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018495 2024-11-24T13:46:58,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018495 2024-11-24T13:46:58,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018495) roll requested 2024-11-24T13:46:58,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018503 2024-11-24T13:46:58,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018495 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018503 2024-11-24T13:46:58,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018503) roll requested 2024-11-24T13:46:58,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018508 2024-11-24T13:46:58,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018508 2024-11-24T13:46:58,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018508) roll requested 2024-11-24T13:46:58,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018515 2024-11-24T13:46:58,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018515 2024-11-24T13:46:58,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018515) roll requested 2024-11-24T13:46:58,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018524 2024-11-24T13:46:58,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018524 2024-11-24T13:46:58,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018524) roll requested 2024-11-24T13:46:58,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018533 2024-11-24T13:46:58,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018533 2024-11-24T13:46:58,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018533) roll requested 2024-11-24T13:46:58,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018539 2024-11-24T13:46:58,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018539 2024-11-24T13:46:58,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018539) roll requested 2024-11-24T13:46:58,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018546 2024-11-24T13:46:58,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018546 2024-11-24T13:46:58,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018546) roll requested 2024-11-24T13:46:58,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018553 2024-11-24T13:46:58,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018553 2024-11-24T13:46:58,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018553) roll requested 2024-11-24T13:46:58,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018559 2024-11-24T13:46:58,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018559 2024-11-24T13:46:58,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018559) roll requested 2024-11-24T13:46:58,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018565 2024-11-24T13:46:58,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018565 2024-11-24T13:46:58,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018565) roll requested 2024-11-24T13:46:58,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018572 2024-11-24T13:46:58,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018572 2024-11-24T13:46:58,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018572) roll requested 2024-11-24T13:46:58,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018578 2024-11-24T13:46:58,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018578 2024-11-24T13:46:58,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018578) roll requested 2024-11-24T13:46:58,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018584 2024-11-24T13:46:58,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018584 2024-11-24T13:46:58,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018584) roll requested 2024-11-24T13:46:58,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018593 2024-11-24T13:46:58,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018593 2024-11-24T13:46:58,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018593) roll requested 2024-11-24T13:46:58,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018599 2024-11-24T13:46:58,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018599 2024-11-24T13:46:58,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018599) roll requested 2024-11-24T13:46:58,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018604 2024-11-24T13:46:58,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018604 2024-11-24T13:46:58,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018604) roll requested 2024-11-24T13:46:58,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018611 2024-11-24T13:46:58,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018611 2024-11-24T13:46:58,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018611) roll requested 2024-11-24T13:46:58,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018619 2024-11-24T13:46:58,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018611 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018619 2024-11-24T13:46:58,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018619) roll requested 2024-11-24T13:46:58,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018627 2024-11-24T13:46:58,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018627 2024-11-24T13:46:58,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018627) roll requested 2024-11-24T13:46:58,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018637 2024-11-24T13:46:58,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018637 2024-11-24T13:46:58,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018637) roll requested 2024-11-24T13:46:58,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018651 2024-11-24T13:46:58,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018651 2024-11-24T13:46:58,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018651) roll requested 2024-11-24T13:46:58,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018659 2024-11-24T13:46:58,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018659 2024-11-24T13:46:58,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,666 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018659) roll requested 2024-11-24T13:46:58,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018667 2024-11-24T13:46:58,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018667 2024-11-24T13:46:58,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018667) roll requested 2024-11-24T13:46:58,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018676 2024-11-24T13:46:58,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018676 2024-11-24T13:46:58,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018676) roll requested 2024-11-24T13:46:58,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018686 2024-11-24T13:46:58,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018686 2024-11-24T13:46:58,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018686) roll requested 2024-11-24T13:46:58,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018697 2024-11-24T13:46:58,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018697 2024-11-24T13:46:58,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018697) roll requested 2024-11-24T13:46:58,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018704 2024-11-24T13:46:58,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018704 2024-11-24T13:46:58,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018704) roll requested 2024-11-24T13:46:58,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018711 2024-11-24T13:46:58,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018711 2024-11-24T13:46:58,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018711) roll requested 2024-11-24T13:46:58,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018716 2024-11-24T13:46:58,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018716 2024-11-24T13:46:58,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,723 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018716) roll requested 2024-11-24T13:46:58,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018723 2024-11-24T13:46:58,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018716 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018723 2024-11-24T13:46:58,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,730 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018723) roll requested 2024-11-24T13:46:58,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018730 2024-11-24T13:46:58,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018723 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018730 2024-11-24T13:46:58,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,738 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018730) roll requested 2024-11-24T13:46:58,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018738 2024-11-24T13:46:58,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018738 2024-11-24T13:46:58,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018738) roll requested 2024-11-24T13:46:58,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018747 2024-11-24T13:46:58,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018738 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018747 2024-11-24T13:46:58,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018747) roll requested 2024-11-24T13:46:58,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018753 2024-11-24T13:46:58,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018753 2024-11-24T13:46:58,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018753) roll requested 2024-11-24T13:46:58,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018759 2024-11-24T13:46:58,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018759 2024-11-24T13:46:58,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018759) roll requested 2024-11-24T13:46:58,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018767 2024-11-24T13:46:58,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018767 2024-11-24T13:46:58,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018767) roll requested 2024-11-24T13:46:58,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018783 2024-11-24T13:46:58,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018783 2024-11-24T13:46:58,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018783) roll requested 2024-11-24T13:46:58,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018791 2024-11-24T13:46:58,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018791 2024-11-24T13:46:58,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018791) roll requested 2024-11-24T13:46:58,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018799 2024-11-24T13:46:58,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018799 2024-11-24T13:46:58,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018799) roll requested 2024-11-24T13:46:58,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018805 2024-11-24T13:46:58,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018805 2024-11-24T13:46:58,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018805) roll requested 2024-11-24T13:46:58,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018814 2024-11-24T13:46:58,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018814 2024-11-24T13:46:58,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018814) roll requested 2024-11-24T13:46:58,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018827 2024-11-24T13:46:58,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018827 2024-11-24T13:46:58,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018827) roll requested 2024-11-24T13:46:58,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018834 2024-11-24T13:46:58,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018834 2024-11-24T13:46:58,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018834) roll requested 2024-11-24T13:46:58,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018841 2024-11-24T13:46:58,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018841 2024-11-24T13:46:58,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018841) roll requested 2024-11-24T13:46:58,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018848 2024-11-24T13:46:58,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018848 2024-11-24T13:46:58,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018848) roll requested 2024-11-24T13:46:58,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018854 2024-11-24T13:46:58,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018854 2024-11-24T13:46:58,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,863 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018854) roll requested 2024-11-24T13:46:58,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018863 2024-11-24T13:46:58,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018863 2024-11-24T13:46:58,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018863) roll requested 2024-11-24T13:46:58,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018874 2024-11-24T13:46:58,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018874 2024-11-24T13:46:58,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018874) roll requested 2024-11-24T13:46:58,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018881 2024-11-24T13:46:58,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018881 2024-11-24T13:46:58,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,891 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018881) roll requested 2024-11-24T13:46:58,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018892 2024-11-24T13:46:58,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018892 2024-11-24T13:46:58,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,897 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018892) roll requested 2024-11-24T13:46:58,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018897 2024-11-24T13:46:58,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018892 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018897 2024-11-24T13:46:58,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,903 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018897) roll requested 2024-11-24T13:46:58,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018903 2024-11-24T13:46:58,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018897 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018903 2024-11-24T13:46:58,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018903) roll requested 2024-11-24T13:46:58,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018910 2024-11-24T13:46:58,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018910 2024-11-24T13:46:58,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018910) roll requested 2024-11-24T13:46:58,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018920 2024-11-24T13:46:58,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018910 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018920 2024-11-24T13:46:58,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,930 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018920) roll requested 2024-11-24T13:46:58,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018931 2024-11-24T13:46:58,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018931 2024-11-24T13:46:58,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018931) roll requested 2024-11-24T13:46:58,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018937 2024-11-24T13:46:58,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018937 2024-11-24T13:46:58,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018937) roll requested 2024-11-24T13:46:58,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018943 2024-11-24T13:46:58,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018943 2024-11-24T13:46:58,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,950 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018943) roll requested 2024-11-24T13:46:58,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018950 2024-11-24T13:46:58,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018943 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018950 2024-11-24T13:46:58,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018950) roll requested 2024-11-24T13:46:58,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018959 2024-11-24T13:46:58,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018959 2024-11-24T13:46:58,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018959) roll requested 2024-11-24T13:46:58,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018969 2024-11-24T13:46:58,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018969 2024-11-24T13:46:58,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018969) roll requested 2024-11-24T13:46:58,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018981 2024-11-24T13:46:58,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018981 2024-11-24T13:46:58,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,990 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018981) roll requested 2024-11-24T13:46:58,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456018991 2024-11-24T13:46:58,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:58,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018991 2024-11-24T13:46:58,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:58,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:58,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456018991) roll requested 2024-11-24T13:46:59,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019000 2024-11-24T13:46:59,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456018991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019000 2024-11-24T13:46:59,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019000) roll requested 2024-11-24T13:46:59,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019008 2024-11-24T13:46:59,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019008 2024-11-24T13:46:59,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019008) roll requested 2024-11-24T13:46:59,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019021 2024-11-24T13:46:59,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019021 2024-11-24T13:46:59,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019021) roll requested 2024-11-24T13:46:59,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019033 2024-11-24T13:46:59,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019033 2024-11-24T13:46:59,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019033) roll requested 2024-11-24T13:46:59,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019048 2024-11-24T13:46:59,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019048 2024-11-24T13:46:59,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019048) roll requested 2024-11-24T13:46:59,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019058 2024-11-24T13:46:59,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019058 2024-11-24T13:46:59,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019058) roll requested 2024-11-24T13:46:59,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019066 2024-11-24T13:46:59,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019058 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019066 2024-11-24T13:46:59,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019066) roll requested 2024-11-24T13:46:59,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019075 2024-11-24T13:46:59,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019066 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019075 2024-11-24T13:46:59,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,081 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019075) roll requested 2024-11-24T13:46:59,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019082 2024-11-24T13:46:59,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019082 2024-11-24T13:46:59,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019082) roll requested 2024-11-24T13:46:59,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019089 2024-11-24T13:46:59,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019089 2024-11-24T13:46:59,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,097 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019089) roll requested 2024-11-24T13:46:59,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019097 2024-11-24T13:46:59,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019097 2024-11-24T13:46:59,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019097) roll requested 2024-11-24T13:46:59,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019104 2024-11-24T13:46:59,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019104 2024-11-24T13:46:59,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019104) roll requested 2024-11-24T13:46:59,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019112 2024-11-24T13:46:59,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019104 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019112 2024-11-24T13:46:59,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019112) roll requested 2024-11-24T13:46:59,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019120 2024-11-24T13:46:59,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019120 2024-11-24T13:46:59,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019120) roll requested 2024-11-24T13:46:59,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019128 2024-11-24T13:46:59,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019128 2024-11-24T13:46:59,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019128) roll requested 2024-11-24T13:46:59,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019136 2024-11-24T13:46:59,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019136 2024-11-24T13:46:59,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019136) roll requested 2024-11-24T13:46:59,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019157 2024-11-24T13:46:59,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019157 2024-11-24T13:46:59,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,167 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019157) roll requested 2024-11-24T13:46:59,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019167 2024-11-24T13:46:59,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019167 2024-11-24T13:46:59,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019167) roll requested 2024-11-24T13:46:59,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019179 2024-11-24T13:46:59,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019179 2024-11-24T13:46:59,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019179) roll requested 2024-11-24T13:46:59,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019188 2024-11-24T13:46:59,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019188 2024-11-24T13:46:59,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019188) roll requested 2024-11-24T13:46:59,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019196 2024-11-24T13:46:59,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019196 2024-11-24T13:46:59,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,209 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019196) roll requested 2024-11-24T13:46:59,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019209 2024-11-24T13:46:59,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019209 2024-11-24T13:46:59,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019209) roll requested 2024-11-24T13:46:59,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019219 2024-11-24T13:46:59,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019219 2024-11-24T13:46:59,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019219) roll requested 2024-11-24T13:46:59,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019227 2024-11-24T13:46:59,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019227 2024-11-24T13:46:59,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019227) roll requested 2024-11-24T13:46:59,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019233 2024-11-24T13:46:59,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019233 2024-11-24T13:46:59,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019233) roll requested 2024-11-24T13:46:59,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019239 2024-11-24T13:46:59,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019239 2024-11-24T13:46:59,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019239) roll requested 2024-11-24T13:46:59,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019247 2024-11-24T13:46:59,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019247 2024-11-24T13:46:59,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019247) roll requested 2024-11-24T13:46:59,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019255 2024-11-24T13:46:59,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019255 2024-11-24T13:46:59,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019255) roll requested 2024-11-24T13:46:59,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019263 2024-11-24T13:46:59,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019263 2024-11-24T13:46:59,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019263) roll requested 2024-11-24T13:46:59,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019275 2024-11-24T13:46:59,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019275 2024-11-24T13:46:59,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,290 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019275) roll requested 2024-11-24T13:46:59,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019291 2024-11-24T13:46:59,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019291 2024-11-24T13:46:59,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019291) roll requested 2024-11-24T13:46:59,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019307 2024-11-24T13:46:59,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019291 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019307 2024-11-24T13:46:59,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019307) roll requested 2024-11-24T13:46:59,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019326 2024-11-24T13:46:59,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019326 2024-11-24T13:46:59,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,335 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019326) roll requested 2024-11-24T13:46:59,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019335 2024-11-24T13:46:59,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019335 2024-11-24T13:46:59,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,342 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019335) roll requested 2024-11-24T13:46:59,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019343 2024-11-24T13:46:59,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019343 2024-11-24T13:46:59,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019343) roll requested 2024-11-24T13:46:59,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019351 2024-11-24T13:46:59,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019351 2024-11-24T13:46:59,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,358 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019351) roll requested 2024-11-24T13:46:59,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019358 2024-11-24T13:46:59,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019351 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019358 2024-11-24T13:46:59,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,367 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019358) roll requested 2024-11-24T13:46:59,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019367 2024-11-24T13:46:59,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019358 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019367 2024-11-24T13:46:59,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019367) roll requested 2024-11-24T13:46:59,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019375 2024-11-24T13:46:59,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019367 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019375 2024-11-24T13:46:59,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019375) roll requested 2024-11-24T13:46:59,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019382 2024-11-24T13:46:59,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019382 2024-11-24T13:46:59,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,388 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019382) roll requested 2024-11-24T13:46:59,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019388 2024-11-24T13:46:59,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019388 2024-11-24T13:46:59,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019388) roll requested 2024-11-24T13:46:59,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019394 2024-11-24T13:46:59,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019394 2024-11-24T13:46:59,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019394) roll requested 2024-11-24T13:46:59,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019400 2024-11-24T13:46:59,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019400 2024-11-24T13:46:59,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019400) roll requested 2024-11-24T13:46:59,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019409 2024-11-24T13:46:59,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019409 2024-11-24T13:46:59,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019409) roll requested 2024-11-24T13:46:59,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019415 2024-11-24T13:46:59,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019415 2024-11-24T13:46:59,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019415) roll requested 2024-11-24T13:46:59,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019421 2024-11-24T13:46:59,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019421 2024-11-24T13:46:59,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019421) roll requested 2024-11-24T13:46:59,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019431 2024-11-24T13:46:59,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019431 2024-11-24T13:46:59,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019431) roll requested 2024-11-24T13:46:59,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019437 2024-11-24T13:46:59,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019437 2024-11-24T13:46:59,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,443 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019437) roll requested 2024-11-24T13:46:59,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019443 2024-11-24T13:46:59,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019443 2024-11-24T13:46:59,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019443) roll requested 2024-11-24T13:46:59,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019452 2024-11-24T13:46:59,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019443 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019452 2024-11-24T13:46:59,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019452) roll requested 2024-11-24T13:46:59,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019460 2024-11-24T13:46:59,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019460 2024-11-24T13:46:59,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019460) roll requested 2024-11-24T13:46:59,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019467 2024-11-24T13:46:59,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019467 2024-11-24T13:46:59,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019467) roll requested 2024-11-24T13:46:59,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019474 2024-11-24T13:46:59,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019474 2024-11-24T13:46:59,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019474) roll requested 2024-11-24T13:46:59,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019481 2024-11-24T13:46:59,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019481 2024-11-24T13:46:59,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019481) roll requested 2024-11-24T13:46:59,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019487 2024-11-24T13:46:59,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019487 2024-11-24T13:46:59,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019487) roll requested 2024-11-24T13:46:59,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019493 2024-11-24T13:46:59,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019493 2024-11-24T13:46:59,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,499 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019493) roll requested 2024-11-24T13:46:59,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019500 2024-11-24T13:46:59,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019500 2024-11-24T13:46:59,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,505 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019500) roll requested 2024-11-24T13:46:59,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019506 2024-11-24T13:46:59,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019506 2024-11-24T13:46:59,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019506) roll requested 2024-11-24T13:46:59,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019512 2024-11-24T13:46:59,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019512 2024-11-24T13:46:59,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,518 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019512) roll requested 2024-11-24T13:46:59,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019518 2024-11-24T13:46:59,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019518 2024-11-24T13:46:59,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019518) roll requested 2024-11-24T13:46:59,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019524 2024-11-24T13:46:59,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019518 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019524 2024-11-24T13:46:59,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019524) roll requested 2024-11-24T13:46:59,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019530 2024-11-24T13:46:59,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019530 2024-11-24T13:46:59,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019530) roll requested 2024-11-24T13:46:59,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019535 2024-11-24T13:46:59,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019535 2024-11-24T13:46:59,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019535) roll requested 2024-11-24T13:46:59,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019541 2024-11-24T13:46:59,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019541 2024-11-24T13:46:59,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019541) roll requested 2024-11-24T13:46:59,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019548 2024-11-24T13:46:59,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019548 2024-11-24T13:46:59,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019548) roll requested 2024-11-24T13:46:59,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019556 2024-11-24T13:46:59,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019556 2024-11-24T13:46:59,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019556) roll requested 2024-11-24T13:46:59,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019566 2024-11-24T13:46:59,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019566 2024-11-24T13:46:59,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,573 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019566) roll requested 2024-11-24T13:46:59,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019574 2024-11-24T13:46:59,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019574 2024-11-24T13:46:59,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019574) roll requested 2024-11-24T13:46:59,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019580 2024-11-24T13:46:59,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019574 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019580 2024-11-24T13:46:59,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019580) roll requested 2024-11-24T13:46:59,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019588 2024-11-24T13:46:59,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019588 2024-11-24T13:46:59,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019588) roll requested 2024-11-24T13:46:59,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019596 2024-11-24T13:46:59,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019596 2024-11-24T13:46:59,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019596) roll requested 2024-11-24T13:46:59,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019607 2024-11-24T13:46:59,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019607 2024-11-24T13:46:59,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019607) roll requested 2024-11-24T13:46:59,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019618 2024-11-24T13:46:59,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019607 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019618 2024-11-24T13:46:59,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019618) roll requested 2024-11-24T13:46:59,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019629 2024-11-24T13:46:59,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019629 2024-11-24T13:46:59,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019629) roll requested 2024-11-24T13:46:59,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019643 2024-11-24T13:46:59,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019643 2024-11-24T13:46:59,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,658 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019643) roll requested 2024-11-24T13:46:59,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019659 2024-11-24T13:46:59,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019643 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019659 2024-11-24T13:46:59,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019659) roll requested 2024-11-24T13:46:59,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019666 2024-11-24T13:46:59,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019666 2024-11-24T13:46:59,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,671 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019666) roll requested 2024-11-24T13:46:59,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019671 2024-11-24T13:46:59,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019666 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019671 2024-11-24T13:46:59,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019671) roll requested 2024-11-24T13:46:59,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019677 2024-11-24T13:46:59,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019671 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019677 2024-11-24T13:46:59,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019677) roll requested 2024-11-24T13:46:59,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019685 2024-11-24T13:46:59,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019685 2024-11-24T13:46:59,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019685) roll requested 2024-11-24T13:46:59,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019694 2024-11-24T13:46:59,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019694 2024-11-24T13:46:59,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019694) roll requested 2024-11-24T13:46:59,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019702 2024-11-24T13:46:59,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019702 2024-11-24T13:46:59,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019702) roll requested 2024-11-24T13:46:59,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019711 2024-11-24T13:46:59,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019711 2024-11-24T13:46:59,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019711) roll requested 2024-11-24T13:46:59,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019719 2024-11-24T13:46:59,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019719 2024-11-24T13:46:59,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019719) roll requested 2024-11-24T13:46:59,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019728 2024-11-24T13:46:59,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019728 2024-11-24T13:46:59,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019728) roll requested 2024-11-24T13:46:59,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019735 2024-11-24T13:46:59,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019735 2024-11-24T13:46:59,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019735) roll requested 2024-11-24T13:46:59,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019747 2024-11-24T13:46:59,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019747 2024-11-24T13:46:59,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,762 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019747) roll requested 2024-11-24T13:46:59,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019763 2024-11-24T13:46:59,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019763 2024-11-24T13:46:59,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019763) roll requested 2024-11-24T13:46:59,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019770 2024-11-24T13:46:59,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019770 2024-11-24T13:46:59,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019770) roll requested 2024-11-24T13:46:59,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019778 2024-11-24T13:46:59,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019778 2024-11-24T13:46:59,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,794 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019778) roll requested 2024-11-24T13:46:59,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019794 2024-11-24T13:46:59,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019794 2024-11-24T13:46:59,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019794) roll requested 2024-11-24T13:46:59,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019801 2024-11-24T13:46:59,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019801 2024-11-24T13:46:59,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019801) roll requested 2024-11-24T13:46:59,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019814 2024-11-24T13:46:59,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019801 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019814 2024-11-24T13:46:59,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019814) roll requested 2024-11-24T13:46:59,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019827 2024-11-24T13:46:59,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019827 2024-11-24T13:46:59,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,838 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019827) roll requested 2024-11-24T13:46:59,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019838 2024-11-24T13:46:59,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019838 2024-11-24T13:46:59,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019838) roll requested 2024-11-24T13:46:59,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019847 2024-11-24T13:46:59,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019838 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019847 2024-11-24T13:46:59,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019847) roll requested 2024-11-24T13:46:59,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019854 2024-11-24T13:46:59,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019854 2024-11-24T13:46:59,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019854) roll requested 2024-11-24T13:46:59,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019860 2024-11-24T13:46:59,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019860 2024-11-24T13:46:59,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019860) roll requested 2024-11-24T13:46:59,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019867 2024-11-24T13:46:59,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019867 2024-11-24T13:46:59,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019867) roll requested 2024-11-24T13:46:59,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019875 2024-11-24T13:46:59,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019875 2024-11-24T13:46:59,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019875) roll requested 2024-11-24T13:46:59,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019887 2024-11-24T13:46:59,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019875 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019887 2024-11-24T13:46:59,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019887) roll requested 2024-11-24T13:46:59,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019893 2024-11-24T13:46:59,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019893 2024-11-24T13:46:59,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019893) roll requested 2024-11-24T13:46:59,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019898 2024-11-24T13:46:59,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019898 2024-11-24T13:46:59,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,903 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019898) roll requested 2024-11-24T13:46:59,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019903 2024-11-24T13:46:59,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019903 2024-11-24T13:46:59,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019903) roll requested 2024-11-24T13:46:59,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019911 2024-11-24T13:46:59,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019911 2024-11-24T13:46:59,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019911) roll requested 2024-11-24T13:46:59,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019921 2024-11-24T13:46:59,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019921 2024-11-24T13:46:59,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019921) roll requested 2024-11-24T13:46:59,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019928 2024-11-24T13:46:59,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019928 2024-11-24T13:46:59,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019928) roll requested 2024-11-24T13:46:59,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019939 2024-11-24T13:46:59,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019939 2024-11-24T13:46:59,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019939) roll requested 2024-11-24T13:46:59,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019945 2024-11-24T13:46:59,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019945 2024-11-24T13:46:59,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,950 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019945) roll requested 2024-11-24T13:46:59,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019950 2024-11-24T13:46:59,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019950 2024-11-24T13:46:59,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019950) roll requested 2024-11-24T13:46:59,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019958 2024-11-24T13:46:59,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019958 2024-11-24T13:46:59,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,967 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019958) roll requested 2024-11-24T13:46:59,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019967 2024-11-24T13:46:59,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019967 2024-11-24T13:46:59,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,975 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019967) roll requested 2024-11-24T13:46:59,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019976 2024-11-24T13:46:59,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019976 2024-11-24T13:46:59,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019976) roll requested 2024-11-24T13:46:59,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019983 2024-11-24T13:46:59,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019983 2024-11-24T13:46:59,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019983) roll requested 2024-11-24T13:46:59,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019989 2024-11-24T13:46:59,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:46:59,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019989 2024-11-24T13:46:59,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:46:59,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:46:59,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019989) roll requested 2024-11-24T13:46:59,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456019995 2024-11-24T13:47:00,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019995 2024-11-24T13:47:00,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456019995) roll requested 2024-11-24T13:47:00,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020004 2024-11-24T13:47:00,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456019995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020004 2024-11-24T13:47:00,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020004) roll requested 2024-11-24T13:47:00,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020009 2024-11-24T13:47:00,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020009 2024-11-24T13:47:00,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,015 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020009) roll requested 2024-11-24T13:47:00,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020015 2024-11-24T13:47:00,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020015 2024-11-24T13:47:00,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020015) roll requested 2024-11-24T13:47:00,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020023 2024-11-24T13:47:00,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020015 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020023 2024-11-24T13:47:00,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020023) roll requested 2024-11-24T13:47:00,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020030 2024-11-24T13:47:00,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020030 2024-11-24T13:47:00,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020030) roll requested 2024-11-24T13:47:00,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020043 2024-11-24T13:47:00,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020030 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020043 2024-11-24T13:47:00,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020043) roll requested 2024-11-24T13:47:00,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020051 2024-11-24T13:47:00,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020051 2024-11-24T13:47:00,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020051) roll requested 2024-11-24T13:47:00,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020059 2024-11-24T13:47:00,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020059 2024-11-24T13:47:00,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020059) roll requested 2024-11-24T13:47:00,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020067 2024-11-24T13:47:00,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020067 2024-11-24T13:47:00,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020067) roll requested 2024-11-24T13:47:00,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020078 2024-11-24T13:47:00,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020067 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020078 2024-11-24T13:47:00,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020078) roll requested 2024-11-24T13:47:00,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020085 2024-11-24T13:47:00,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020078 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020085 2024-11-24T13:47:00,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020085) roll requested 2024-11-24T13:47:00,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020093 2024-11-24T13:47:00,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020093 2024-11-24T13:47:00,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020093) roll requested 2024-11-24T13:47:00,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020103 2024-11-24T13:47:00,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020103 2024-11-24T13:47:00,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020103) roll requested 2024-11-24T13:47:00,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020110 2024-11-24T13:47:00,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020110 2024-11-24T13:47:00,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020110) roll requested 2024-11-24T13:47:00,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020116 2024-11-24T13:47:00,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020110 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020116 2024-11-24T13:47:00,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020116) roll requested 2024-11-24T13:47:00,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020123 2024-11-24T13:47:00,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020123 2024-11-24T13:47:00,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020123) roll requested 2024-11-24T13:47:00,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020131 2024-11-24T13:47:00,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020131 2024-11-24T13:47:00,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020131) roll requested 2024-11-24T13:47:00,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020137 2024-11-24T13:47:00,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020137 2024-11-24T13:47:00,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020137) roll requested 2024-11-24T13:47:00,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020144 2024-11-24T13:47:00,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020144 2024-11-24T13:47:00,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020144) roll requested 2024-11-24T13:47:00,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020151 2024-11-24T13:47:00,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020144 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020151 2024-11-24T13:47:00,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020151) roll requested 2024-11-24T13:47:00,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020158 2024-11-24T13:47:00,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020158 2024-11-24T13:47:00,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020158) roll requested 2024-11-24T13:47:00,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020165 2024-11-24T13:47:00,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020165 2024-11-24T13:47:00,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020165) roll requested 2024-11-24T13:47:00,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020171 2024-11-24T13:47:00,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020171 2024-11-24T13:47:00,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020171) roll requested 2024-11-24T13:47:00,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020179 2024-11-24T13:47:00,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020179 2024-11-24T13:47:00,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020179) roll requested 2024-11-24T13:47:00,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020185 2024-11-24T13:47:00,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020185 2024-11-24T13:47:00,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020185) roll requested 2024-11-24T13:47:00,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020192 2024-11-24T13:47:00,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020192 2024-11-24T13:47:00,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020192) roll requested 2024-11-24T13:47:00,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020203 2024-11-24T13:47:00,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020203 2024-11-24T13:47:00,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020203) roll requested 2024-11-24T13:47:00,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020211 2024-11-24T13:47:00,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020203 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020211 2024-11-24T13:47:00,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,217 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020211) roll requested 2024-11-24T13:47:00,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020218 2024-11-24T13:47:00,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020211 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020218 2024-11-24T13:47:00,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020218) roll requested 2024-11-24T13:47:00,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020223 2024-11-24T13:47:00,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020218 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020223 2024-11-24T13:47:00,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020223) roll requested 2024-11-24T13:47:00,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020230 2024-11-24T13:47:00,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020230 2024-11-24T13:47:00,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020230) roll requested 2024-11-24T13:47:00,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020238 2024-11-24T13:47:00,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020238 2024-11-24T13:47:00,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020238) roll requested 2024-11-24T13:47:00,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020246 2024-11-24T13:47:00,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020246 2024-11-24T13:47:00,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020246) roll requested 2024-11-24T13:47:00,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020255 2024-11-24T13:47:00,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020255 2024-11-24T13:47:00,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020255) roll requested 2024-11-24T13:47:00,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020269 2024-11-24T13:47:00,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020269 2024-11-24T13:47:00,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020269) roll requested 2024-11-24T13:47:00,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020274 2024-11-24T13:47:00,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020274 2024-11-24T13:47:00,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020274) roll requested 2024-11-24T13:47:00,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020280 2024-11-24T13:47:00,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020280 2024-11-24T13:47:00,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020280) roll requested 2024-11-24T13:47:00,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020286 2024-11-24T13:47:00,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020286 2024-11-24T13:47:00,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020286) roll requested 2024-11-24T13:47:00,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020295 2024-11-24T13:47:00,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020295 2024-11-24T13:47:00,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,302 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020295) roll requested 2024-11-24T13:47:00,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020303 2024-11-24T13:47:00,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020303 2024-11-24T13:47:00,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020303) roll requested 2024-11-24T13:47:00,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020311 2024-11-24T13:47:00,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020303 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020311 2024-11-24T13:47:00,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020311) roll requested 2024-11-24T13:47:00,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020325 2024-11-24T13:47:00,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020311 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020325 2024-11-24T13:47:00,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020325) roll requested 2024-11-24T13:47:00,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020334 2024-11-24T13:47:00,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020334 2024-11-24T13:47:00,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020334) roll requested 2024-11-24T13:47:00,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020340 2024-11-24T13:47:00,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020340 2024-11-24T13:47:00,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020340) roll requested 2024-11-24T13:47:00,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020346 2024-11-24T13:47:00,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020346 2024-11-24T13:47:00,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020346) roll requested 2024-11-24T13:47:00,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020353 2024-11-24T13:47:00,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020353 2024-11-24T13:47:00,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020353) roll requested 2024-11-24T13:47:00,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020359 2024-11-24T13:47:00,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020359 2024-11-24T13:47:00,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,367 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020359) roll requested 2024-11-24T13:47:00,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020367 2024-11-24T13:47:00,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020367 2024-11-24T13:47:00,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,381 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020367) roll requested 2024-11-24T13:47:00,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020381 2024-11-24T13:47:00,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020367 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020381 2024-11-24T13:47:00,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020381) roll requested 2024-11-24T13:47:00,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020395 2024-11-24T13:47:00,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020381 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020395 2024-11-24T13:47:00,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020395) roll requested 2024-11-24T13:47:00,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020406 2024-11-24T13:47:00,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020406 2024-11-24T13:47:00,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020406) roll requested 2024-11-24T13:47:00,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020413 2024-11-24T13:47:00,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020413 2024-11-24T13:47:00,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,418 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020413) roll requested 2024-11-24T13:47:00,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020418 2024-11-24T13:47:00,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020418 2024-11-24T13:47:00,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,425 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020418) roll requested 2024-11-24T13:47:00,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020426 2024-11-24T13:47:00,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020418 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020426 2024-11-24T13:47:00,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020426) roll requested 2024-11-24T13:47:00,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020431 2024-11-24T13:47:00,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020426 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020431 2024-11-24T13:47:00,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020431) roll requested 2024-11-24T13:47:00,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020438 2024-11-24T13:47:00,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020438 2024-11-24T13:47:00,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,443 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020438) roll requested 2024-11-24T13:47:00,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020444 2024-11-24T13:47:00,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020444 2024-11-24T13:47:00,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020444) roll requested 2024-11-24T13:47:00,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020449 2024-11-24T13:47:00,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020449 2024-11-24T13:47:00,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,454 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020449) roll requested 2024-11-24T13:47:00,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020454 2024-11-24T13:47:00,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020454 2024-11-24T13:47:00,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020454) roll requested 2024-11-24T13:47:00,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020460 2024-11-24T13:47:00,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020460 2024-11-24T13:47:00,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020460) roll requested 2024-11-24T13:47:00,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020466 2024-11-24T13:47:00,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020466 2024-11-24T13:47:00,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020466) roll requested 2024-11-24T13:47:00,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020471 2024-11-24T13:47:00,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020471 2024-11-24T13:47:00,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020471) roll requested 2024-11-24T13:47:00,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020477 2024-11-24T13:47:00,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020477 2024-11-24T13:47:00,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020477) roll requested 2024-11-24T13:47:00,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020483 2024-11-24T13:47:00,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020483 2024-11-24T13:47:00,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020483) roll requested 2024-11-24T13:47:00,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020489 2024-11-24T13:47:00,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020489 2024-11-24T13:47:00,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020489) roll requested 2024-11-24T13:47:00,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020494 2024-11-24T13:47:00,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020494 2024-11-24T13:47:00,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,499 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020494) roll requested 2024-11-24T13:47:00,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020500 2024-11-24T13:47:00,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020500 2024-11-24T13:47:00,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020500) roll requested 2024-11-24T13:47:00,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020508 2024-11-24T13:47:00,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020508 2024-11-24T13:47:00,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020508) roll requested 2024-11-24T13:47:00,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020513 2024-11-24T13:47:00,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020513 2024-11-24T13:47:00,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020513) roll requested 2024-11-24T13:47:00,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020527 2024-11-24T13:47:00,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020527 2024-11-24T13:47:00,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020527) roll requested 2024-11-24T13:47:00,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020533 2024-11-24T13:47:00,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020533 2024-11-24T13:47:00,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020533) roll requested 2024-11-24T13:47:00,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020540 2024-11-24T13:47:00,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020540 2024-11-24T13:47:00,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020540) roll requested 2024-11-24T13:47:00,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020546 2024-11-24T13:47:00,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020540 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020546 2024-11-24T13:47:00,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020546) roll requested 2024-11-24T13:47:00,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020552 2024-11-24T13:47:00,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020552 2024-11-24T13:47:00,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020552) roll requested 2024-11-24T13:47:00,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020558 2024-11-24T13:47:00,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020558 2024-11-24T13:47:00,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020558) roll requested 2024-11-24T13:47:00,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020566 2024-11-24T13:47:00,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020566 2024-11-24T13:47:00,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020566) roll requested 2024-11-24T13:47:00,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020572 2024-11-24T13:47:00,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020572 2024-11-24T13:47:00,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020572) roll requested 2024-11-24T13:47:00,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020577 2024-11-24T13:47:00,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020577 2024-11-24T13:47:00,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,582 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020577) roll requested 2024-11-24T13:47:00,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020583 2024-11-24T13:47:00,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020583 2024-11-24T13:47:00,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020583) roll requested 2024-11-24T13:47:00,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020588 2024-11-24T13:47:00,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020588 2024-11-24T13:47:00,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020588) roll requested 2024-11-24T13:47:00,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020597 2024-11-24T13:47:00,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020597 2024-11-24T13:47:00,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020597) roll requested 2024-11-24T13:47:00,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020603 2024-11-24T13:47:00,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020603 2024-11-24T13:47:00,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020603) roll requested 2024-11-24T13:47:00,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020608 2024-11-24T13:47:00,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020608 2024-11-24T13:47:00,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020608) roll requested 2024-11-24T13:47:00,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020614 2024-11-24T13:47:00,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020614 2024-11-24T13:47:00,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,619 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020614) roll requested 2024-11-24T13:47:00,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020619 2024-11-24T13:47:00,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020619 2024-11-24T13:47:00,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020619) roll requested 2024-11-24T13:47:00,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020625 2024-11-24T13:47:00,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020625 2024-11-24T13:47:00,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,630 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020625) roll requested 2024-11-24T13:47:00,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020630 2024-11-24T13:47:00,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020630 2024-11-24T13:47:00,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,638 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020630) roll requested 2024-11-24T13:47:00,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020638 2024-11-24T13:47:00,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020630 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020638 2024-11-24T13:47:00,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020638) roll requested 2024-11-24T13:47:00,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020647 2024-11-24T13:47:00,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020647 2024-11-24T13:47:00,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020647) roll requested 2024-11-24T13:47:00,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020654 2024-11-24T13:47:00,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020654 2024-11-24T13:47:00,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020654) roll requested 2024-11-24T13:47:00,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020660 2024-11-24T13:47:00,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020660 2024-11-24T13:47:00,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020660) roll requested 2024-11-24T13:47:00,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020666 2024-11-24T13:47:00,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020666 2024-11-24T13:47:00,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020666) roll requested 2024-11-24T13:47:00,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020672 2024-11-24T13:47:00,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020666 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020672 2024-11-24T13:47:00,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020672) roll requested 2024-11-24T13:47:00,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020677 2024-11-24T13:47:00,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020677 2024-11-24T13:47:00,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020677) roll requested 2024-11-24T13:47:00,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020683 2024-11-24T13:47:00,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020683 2024-11-24T13:47:00,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020683) roll requested 2024-11-24T13:47:00,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020691 2024-11-24T13:47:00,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020691 2024-11-24T13:47:00,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020691) roll requested 2024-11-24T13:47:00,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020697 2024-11-24T13:47:00,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020691 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020697 2024-11-24T13:47:00,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020697) roll requested 2024-11-24T13:47:00,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020702 2024-11-24T13:47:00,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020702 2024-11-24T13:47:00,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020702) roll requested 2024-11-24T13:47:00,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020708 2024-11-24T13:47:00,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020708 2024-11-24T13:47:00,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020708) roll requested 2024-11-24T13:47:00,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020713 2024-11-24T13:47:00,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020713 2024-11-24T13:47:00,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020713) roll requested 2024-11-24T13:47:00,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020718 2024-11-24T13:47:00,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020718 2024-11-24T13:47:00,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020718) roll requested 2024-11-24T13:47:00,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020725 2024-11-24T13:47:00,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020718 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020725 2024-11-24T13:47:00,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020725) roll requested 2024-11-24T13:47:00,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020733 2024-11-24T13:47:00,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020733 2024-11-24T13:47:00,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,741 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020733) roll requested 2024-11-24T13:47:00,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020741 2024-11-24T13:47:00,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020741 2024-11-24T13:47:00,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020741) roll requested 2024-11-24T13:47:00,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020751 2024-11-24T13:47:00,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020751 2024-11-24T13:47:00,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020751) roll requested 2024-11-24T13:47:00,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020758 2024-11-24T13:47:00,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020758 2024-11-24T13:47:00,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020758) roll requested 2024-11-24T13:47:00,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020771 2024-11-24T13:47:00,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020771 2024-11-24T13:47:00,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,787 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020771) roll requested 2024-11-24T13:47:00,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020787 2024-11-24T13:47:00,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020787 2024-11-24T13:47:00,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020787) roll requested 2024-11-24T13:47:00,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020799 2024-11-24T13:47:00,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020787 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020799 2024-11-24T13:47:00,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020799) roll requested 2024-11-24T13:47:00,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020807 2024-11-24T13:47:00,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020807 2024-11-24T13:47:00,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020807) roll requested 2024-11-24T13:47:00,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020812 2024-11-24T13:47:00,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020807 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020812 2024-11-24T13:47:00,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,819 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020812) roll requested 2024-11-24T13:47:00,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020819 2024-11-24T13:47:00,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020819 2024-11-24T13:47:00,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020819) roll requested 2024-11-24T13:47:00,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020827 2024-11-24T13:47:00,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020819 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020827 2024-11-24T13:47:00,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020827) roll requested 2024-11-24T13:47:00,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020832 2024-11-24T13:47:00,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020832 2024-11-24T13:47:00,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020832) roll requested 2024-11-24T13:47:00,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020843 2024-11-24T13:47:00,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020843 2024-11-24T13:47:00,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020843) roll requested 2024-11-24T13:47:00,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020849 2024-11-24T13:47:00,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020843 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020849 2024-11-24T13:47:00,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020849) roll requested 2024-11-24T13:47:00,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020856 2024-11-24T13:47:00,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020856 2024-11-24T13:47:00,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020856) roll requested 2024-11-24T13:47:00,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020862 2024-11-24T13:47:00,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020862 2024-11-24T13:47:00,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020862) roll requested 2024-11-24T13:47:00,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020868 2024-11-24T13:47:00,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020868 2024-11-24T13:47:00,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,874 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020868) roll requested 2024-11-24T13:47:00,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020874 2024-11-24T13:47:00,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020874 2024-11-24T13:47:00,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020874) roll requested 2024-11-24T13:47:00,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020880 2024-11-24T13:47:00,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020880 2024-11-24T13:47:00,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,886 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020880) roll requested 2024-11-24T13:47:00,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020886 2024-11-24T13:47:00,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020880 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020886 2024-11-24T13:47:00,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020886) roll requested 2024-11-24T13:47:00,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020893 2024-11-24T13:47:00,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020886 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020893 2024-11-24T13:47:00,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020893) roll requested 2024-11-24T13:47:00,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020900 2024-11-24T13:47:00,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020900 2024-11-24T13:47:00,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020900) roll requested 2024-11-24T13:47:00,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020906 2024-11-24T13:47:00,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020906 2024-11-24T13:47:00,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,913 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020906) roll requested 2024-11-24T13:47:00,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020913 2024-11-24T13:47:00,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020913 2024-11-24T13:47:00,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,923 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020913) roll requested 2024-11-24T13:47:00,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020923 2024-11-24T13:47:00,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020923 2024-11-24T13:47:00,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020923) roll requested 2024-11-24T13:47:00,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020930 2024-11-24T13:47:00,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020923 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020930 2024-11-24T13:47:00,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,938 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020930) roll requested 2024-11-24T13:47:00,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020938 2024-11-24T13:47:00,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020930 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020938 2024-11-24T13:47:00,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020938) roll requested 2024-11-24T13:47:00,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020954 2024-11-24T13:47:00,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020954 2024-11-24T13:47:00,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,962 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020954) roll requested 2024-11-24T13:47:00,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020963 2024-11-24T13:47:00,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020963 2024-11-24T13:47:00,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020963) roll requested 2024-11-24T13:47:00,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020968 2024-11-24T13:47:00,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020963 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020968 2024-11-24T13:47:00,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,975 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020968) roll requested 2024-11-24T13:47:00,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020975 2024-11-24T13:47:00,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020975 2024-11-24T13:47:00,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020975) roll requested 2024-11-24T13:47:00,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020980 2024-11-24T13:47:00,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020975 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020980 2024-11-24T13:47:00,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020980) roll requested 2024-11-24T13:47:00,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020986 2024-11-24T13:47:00,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020986 2024-11-24T13:47:00,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020986) roll requested 2024-11-24T13:47:00,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020991 2024-11-24T13:47:00,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:00,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020991 2024-11-24T13:47:00,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:00,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:00,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020991) roll requested 2024-11-24T13:47:00,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456020999 2024-11-24T13:47:01,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020999 2024-11-24T13:47:01,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456020999) roll requested 2024-11-24T13:47:01,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021007 2024-11-24T13:47:01,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456020999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021007 2024-11-24T13:47:01,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,013 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021007) roll requested 2024-11-24T13:47:01,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021013 2024-11-24T13:47:01,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021013 2024-11-24T13:47:01,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021013) roll requested 2024-11-24T13:47:01,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021018 2024-11-24T13:47:01,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021018 2024-11-24T13:47:01,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,027 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021018) roll requested 2024-11-24T13:47:01,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021027 2024-11-24T13:47:01,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021027 2024-11-24T13:47:01,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021027) roll requested 2024-11-24T13:47:01,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021032 2024-11-24T13:47:01,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021032 2024-11-24T13:47:01,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,037 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021032) roll requested 2024-11-24T13:47:01,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021037 2024-11-24T13:47:01,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021037 2024-11-24T13:47:01,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,042 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021037) roll requested 2024-11-24T13:47:01,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021043 2024-11-24T13:47:01,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021043 2024-11-24T13:47:01,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,050 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021043) roll requested 2024-11-24T13:47:01,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021051 2024-11-24T13:47:01,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021051 2024-11-24T13:47:01,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021051) roll requested 2024-11-24T13:47:01,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021059 2024-11-24T13:47:01,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021059 2024-11-24T13:47:01,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021059) roll requested 2024-11-24T13:47:01,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021065 2024-11-24T13:47:01,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021065 2024-11-24T13:47:01,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021065) roll requested 2024-11-24T13:47:01,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021070 2024-11-24T13:47:01,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021070 2024-11-24T13:47:01,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021070) roll requested 2024-11-24T13:47:01,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021076 2024-11-24T13:47:01,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021076 2024-11-24T13:47:01,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021076) roll requested 2024-11-24T13:47:01,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021083 2024-11-24T13:47:01,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021083 2024-11-24T13:47:01,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,091 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021083) roll requested 2024-11-24T13:47:01,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021091 2024-11-24T13:47:01,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021091 2024-11-24T13:47:01,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021091) roll requested 2024-11-24T13:47:01,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021099 2024-11-24T13:47:01,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021099 2024-11-24T13:47:01,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021099) roll requested 2024-11-24T13:47:01,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021107 2024-11-24T13:47:01,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021107 2024-11-24T13:47:01,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021107) roll requested 2024-11-24T13:47:01,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021112 2024-11-24T13:47:01,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021112 2024-11-24T13:47:01,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,117 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021112) roll requested 2024-11-24T13:47:01,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021117 2024-11-24T13:47:01,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021117 2024-11-24T13:47:01,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021117) roll requested 2024-11-24T13:47:01,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021126 2024-11-24T13:47:01,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021126 2024-11-24T13:47:01,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,131 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021126) roll requested 2024-11-24T13:47:01,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021131 2024-11-24T13:47:01,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021126 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021131 2024-11-24T13:47:01,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021131) roll requested 2024-11-24T13:47:01,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021136 2024-11-24T13:47:01,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021136 2024-11-24T13:47:01,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021136) roll requested 2024-11-24T13:47:01,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021143 2024-11-24T13:47:01,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021143 2024-11-24T13:47:01,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021143) roll requested 2024-11-24T13:47:01,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021149 2024-11-24T13:47:01,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021149 2024-11-24T13:47:01,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021149) roll requested 2024-11-24T13:47:01,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021159 2024-11-24T13:47:01,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021159 2024-11-24T13:47:01,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021159) roll requested 2024-11-24T13:47:01,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021171 2024-11-24T13:47:01,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021171 2024-11-24T13:47:01,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021171) roll requested 2024-11-24T13:47:01,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021180 2024-11-24T13:47:01,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021180 2024-11-24T13:47:01,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021180) roll requested 2024-11-24T13:47:01,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021191 2024-11-24T13:47:01,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021191 2024-11-24T13:47:01,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021191) roll requested 2024-11-24T13:47:01,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021198 2024-11-24T13:47:01,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021198 2024-11-24T13:47:01,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021198) roll requested 2024-11-24T13:47:01,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021204 2024-11-24T13:47:01,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021204 2024-11-24T13:47:01,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021204) roll requested 2024-11-24T13:47:01,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021210 2024-11-24T13:47:01,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021210 2024-11-24T13:47:01,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021210) roll requested 2024-11-24T13:47:01,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021217 2024-11-24T13:47:01,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021217 2024-11-24T13:47:01,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021217) roll requested 2024-11-24T13:47:01,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021226 2024-11-24T13:47:01,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021217 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021226 2024-11-24T13:47:01,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021226) roll requested 2024-11-24T13:47:01,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021231 2024-11-24T13:47:01,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021231 2024-11-24T13:47:01,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021231) roll requested 2024-11-24T13:47:01,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021242 2024-11-24T13:47:01,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021242 2024-11-24T13:47:01,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,250 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021242) roll requested 2024-11-24T13:47:01,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021250 2024-11-24T13:47:01,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021250 2024-11-24T13:47:01,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021250) roll requested 2024-11-24T13:47:01,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021256 2024-11-24T13:47:01,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021250 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021256 2024-11-24T13:47:01,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021256) roll requested 2024-11-24T13:47:01,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021263 2024-11-24T13:47:01,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021263 2024-11-24T13:47:01,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021263) roll requested 2024-11-24T13:47:01,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021270 2024-11-24T13:47:01,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021270 2024-11-24T13:47:01,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021270) roll requested 2024-11-24T13:47:01,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021278 2024-11-24T13:47:01,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021278 2024-11-24T13:47:01,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021278) roll requested 2024-11-24T13:47:01,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021288 2024-11-24T13:47:01,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021288 2024-11-24T13:47:01,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021288) roll requested 2024-11-24T13:47:01,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021294 2024-11-24T13:47:01,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021294 2024-11-24T13:47:01,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021294) roll requested 2024-11-24T13:47:01,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021302 2024-11-24T13:47:01,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021302 2024-11-24T13:47:01,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021302) roll requested 2024-11-24T13:47:01,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021311 2024-11-24T13:47:01,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021311 2024-11-24T13:47:01,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021311) roll requested 2024-11-24T13:47:01,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021317 2024-11-24T13:47:01,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021311 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021317 2024-11-24T13:47:01,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,323 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021317) roll requested 2024-11-24T13:47:01,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021323 2024-11-24T13:47:01,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021323 2024-11-24T13:47:01,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021323) roll requested 2024-11-24T13:47:01,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021329 2024-11-24T13:47:01,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021329 2024-11-24T13:47:01,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021329) roll requested 2024-11-24T13:47:01,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021334 2024-11-24T13:47:01,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021329 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021334 2024-11-24T13:47:01,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021334) roll requested 2024-11-24T13:47:01,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021341 2024-11-24T13:47:01,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021341 2024-11-24T13:47:01,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021341) roll requested 2024-11-24T13:47:01,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021348 2024-11-24T13:47:01,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021348 2024-11-24T13:47:01,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021348) roll requested 2024-11-24T13:47:01,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021355 2024-11-24T13:47:01,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021355 2024-11-24T13:47:01,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021355) roll requested 2024-11-24T13:47:01,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021363 2024-11-24T13:47:01,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021363 2024-11-24T13:47:01,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021363) roll requested 2024-11-24T13:47:01,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021375 2024-11-24T13:47:01,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021375 2024-11-24T13:47:01,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021375) roll requested 2024-11-24T13:47:01,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021386 2024-11-24T13:47:01,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021386 2024-11-24T13:47:01,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021386) roll requested 2024-11-24T13:47:01,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021393 2024-11-24T13:47:01,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021386 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021393 2024-11-24T13:47:01,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,398 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021393) roll requested 2024-11-24T13:47:01,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021398 2024-11-24T13:47:01,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021398 2024-11-24T13:47:01,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021398) roll requested 2024-11-24T13:47:01,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021404 2024-11-24T13:47:01,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021398 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021404 2024-11-24T13:47:01,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021404) roll requested 2024-11-24T13:47:01,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021411 2024-11-24T13:47:01,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021411 2024-11-24T13:47:01,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,418 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021411) roll requested 2024-11-24T13:47:01,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021419 2024-11-24T13:47:01,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021419 2024-11-24T13:47:01,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021419) roll requested 2024-11-24T13:47:01,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021427 2024-11-24T13:47:01,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021427 2024-11-24T13:47:01,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021427) roll requested 2024-11-24T13:47:01,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021434 2024-11-24T13:47:01,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021434 2024-11-24T13:47:01,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021434) roll requested 2024-11-24T13:47:01,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021445 2024-11-24T13:47:01,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021445 2024-11-24T13:47:01,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021445) roll requested 2024-11-24T13:47:01,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021451 2024-11-24T13:47:01,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021451 2024-11-24T13:47:01,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021451) roll requested 2024-11-24T13:47:01,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021457 2024-11-24T13:47:01,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021457 2024-11-24T13:47:01,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021457) roll requested 2024-11-24T13:47:01,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021464 2024-11-24T13:47:01,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021464 2024-11-24T13:47:01,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021464) roll requested 2024-11-24T13:47:01,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021475 2024-11-24T13:47:01,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021464 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021475 2024-11-24T13:47:01,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021475) roll requested 2024-11-24T13:47:01,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021480 2024-11-24T13:47:01,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021480 2024-11-24T13:47:01,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,485 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021480) roll requested 2024-11-24T13:47:01,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021485 2024-11-24T13:47:01,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021485 2024-11-24T13:47:01,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021485) roll requested 2024-11-24T13:47:01,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021491 2024-11-24T13:47:01,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021485 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021491 2024-11-24T13:47:01,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,498 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021491) roll requested 2024-11-24T13:47:01,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021498 2024-11-24T13:47:01,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021498 2024-11-24T13:47:01,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,505 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021498) roll requested 2024-11-24T13:47:01,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021506 2024-11-24T13:47:01,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021498 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021506 2024-11-24T13:47:01,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021506) roll requested 2024-11-24T13:47:01,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021514 2024-11-24T13:47:01,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021514 2024-11-24T13:47:01,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,523 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021514) roll requested 2024-11-24T13:47:01,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021524 2024-11-24T13:47:01,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021524 2024-11-24T13:47:01,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021524) roll requested 2024-11-24T13:47:01,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021533 2024-11-24T13:47:01,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021533 2024-11-24T13:47:01,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021533) roll requested 2024-11-24T13:47:01,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021544 2024-11-24T13:47:01,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021544 2024-11-24T13:47:01,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021544) roll requested 2024-11-24T13:47:01,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021554 2024-11-24T13:47:01,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021554 2024-11-24T13:47:01,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021554) roll requested 2024-11-24T13:47:01,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021564 2024-11-24T13:47:01,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021554 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021564 2024-11-24T13:47:01,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,579 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021564) roll requested 2024-11-24T13:47:01,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021579 2024-11-24T13:47:01,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021579 2024-11-24T13:47:01,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,586 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021579) roll requested 2024-11-24T13:47:01,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021587 2024-11-24T13:47:01,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021579 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021587 2024-11-24T13:47:01,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021587) roll requested 2024-11-24T13:47:01,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021599 2024-11-24T13:47:01,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021599 2024-11-24T13:47:01,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021599) roll requested 2024-11-24T13:47:01,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021611 2024-11-24T13:47:01,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021611 2024-11-24T13:47:01,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021611) roll requested 2024-11-24T13:47:01,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021618 2024-11-24T13:47:01,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021611 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021618 2024-11-24T13:47:01,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021618) roll requested 2024-11-24T13:47:01,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021623 2024-11-24T13:47:01,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021623 2024-11-24T13:47:01,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021623) roll requested 2024-11-24T13:47:01,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021628 2024-11-24T13:47:01,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021628 2024-11-24T13:47:01,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021628) roll requested 2024-11-24T13:47:01,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021634 2024-11-24T13:47:01,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021634 2024-11-24T13:47:01,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021634) roll requested 2024-11-24T13:47:01,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021641 2024-11-24T13:47:01,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021641 2024-11-24T13:47:01,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021641) roll requested 2024-11-24T13:47:01,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021649 2024-11-24T13:47:01,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021649 2024-11-24T13:47:01,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021649) roll requested 2024-11-24T13:47:01,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021658 2024-11-24T13:47:01,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021658 2024-11-24T13:47:01,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021658) roll requested 2024-11-24T13:47:01,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021664 2024-11-24T13:47:01,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021658 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021664 2024-11-24T13:47:01,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021664) roll requested 2024-11-24T13:47:01,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021670 2024-11-24T13:47:01,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021670 2024-11-24T13:47:01,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,678 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021670) roll requested 2024-11-24T13:47:01,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021678 2024-11-24T13:47:01,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021678 2024-11-24T13:47:01,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021678) roll requested 2024-11-24T13:47:01,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021684 2024-11-24T13:47:01,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021684 2024-11-24T13:47:01,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021684) roll requested 2024-11-24T13:47:01,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021689 2024-11-24T13:47:01,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021689 2024-11-24T13:47:01,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021689) roll requested 2024-11-24T13:47:01,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021694 2024-11-24T13:47:01,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021694 2024-11-24T13:47:01,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021694) roll requested 2024-11-24T13:47:01,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021701 2024-11-24T13:47:01,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021701 2024-11-24T13:47:01,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021701) roll requested 2024-11-24T13:47:01,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021708 2024-11-24T13:47:01,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021708 2024-11-24T13:47:01,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021708) roll requested 2024-11-24T13:47:01,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021714 2024-11-24T13:47:01,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021714 2024-11-24T13:47:01,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,723 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021714) roll requested 2024-11-24T13:47:01,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021723 2024-11-24T13:47:01,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021723 2024-11-24T13:47:01,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021723) roll requested 2024-11-24T13:47:01,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021729 2024-11-24T13:47:01,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021723 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021729 2024-11-24T13:47:01,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021729) roll requested 2024-11-24T13:47:01,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021734 2024-11-24T13:47:01,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021734 2024-11-24T13:47:01,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,741 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021734) roll requested 2024-11-24T13:47:01,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021741 2024-11-24T13:47:01,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021741 2024-11-24T13:47:01,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021741) roll requested 2024-11-24T13:47:01,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021750 2024-11-24T13:47:01,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021750 2024-11-24T13:47:01,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021750) roll requested 2024-11-24T13:47:01,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021758 2024-11-24T13:47:01,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021758 2024-11-24T13:47:01,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021758) roll requested 2024-11-24T13:47:01,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021765 2024-11-24T13:47:01,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021765 2024-11-24T13:47:01,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021765) roll requested 2024-11-24T13:47:01,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021770 2024-11-24T13:47:01,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021770 2024-11-24T13:47:01,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,777 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021770) roll requested 2024-11-24T13:47:01,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021777 2024-11-24T13:47:01,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021777 2024-11-24T13:47:01,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021777) roll requested 2024-11-24T13:47:01,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021784 2024-11-24T13:47:01,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021784 2024-11-24T13:47:01,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021784) roll requested 2024-11-24T13:47:01,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021795 2024-11-24T13:47:01,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021795 2024-11-24T13:47:01,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021795) roll requested 2024-11-24T13:47:01,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021807 2024-11-24T13:47:01,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021807 2024-11-24T13:47:01,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,819 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021807) roll requested 2024-11-24T13:47:01,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021819 2024-11-24T13:47:01,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021807 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021819 2024-11-24T13:47:01,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021819) roll requested 2024-11-24T13:47:01,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021828 2024-11-24T13:47:01,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021819 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021828 2024-11-24T13:47:01,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021828) roll requested 2024-11-24T13:47:01,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021835 2024-11-24T13:47:01,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021835 2024-11-24T13:47:01,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,842 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021835) roll requested 2024-11-24T13:47:01,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021842 2024-11-24T13:47:01,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021835 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021842 2024-11-24T13:47:01,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021842) roll requested 2024-11-24T13:47:01,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021851 2024-11-24T13:47:01,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021851 2024-11-24T13:47:01,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021851) roll requested 2024-11-24T13:47:01,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021857 2024-11-24T13:47:01,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021857 2024-11-24T13:47:01,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021857) roll requested 2024-11-24T13:47:01,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021863 2024-11-24T13:47:01,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021863 2024-11-24T13:47:01,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021863) roll requested 2024-11-24T13:47:01,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021871 2024-11-24T13:47:01,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021871 2024-11-24T13:47:01,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021871) roll requested 2024-11-24T13:47:01,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021878 2024-11-24T13:47:01,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021878 2024-11-24T13:47:01,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021878) roll requested 2024-11-24T13:47:01,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021883 2024-11-24T13:47:01,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021883 2024-11-24T13:47:01,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021883) roll requested 2024-11-24T13:47:01,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021890 2024-11-24T13:47:01,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021890 2024-11-24T13:47:01,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021890) roll requested 2024-11-24T13:47:01,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021898 2024-11-24T13:47:01,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021898 2024-11-24T13:47:01,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021898) roll requested 2024-11-24T13:47:01,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021904 2024-11-24T13:47:01,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021904 2024-11-24T13:47:01,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021904) roll requested 2024-11-24T13:47:01,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021912 2024-11-24T13:47:01,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021904 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021912 2024-11-24T13:47:01,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021912) roll requested 2024-11-24T13:47:01,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021918 2024-11-24T13:47:01,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021918 2024-11-24T13:47:01,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021918) roll requested 2024-11-24T13:47:01,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021925 2024-11-24T13:47:01,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021918 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021925 2024-11-24T13:47:01,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021925) roll requested 2024-11-24T13:47:01,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021935 2024-11-24T13:47:01,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021935 2024-11-24T13:47:01,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021935) roll requested 2024-11-24T13:47:01,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021947 2024-11-24T13:47:01,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021935 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021947 2024-11-24T13:47:01,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021947) roll requested 2024-11-24T13:47:01,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021960 2024-11-24T13:47:01,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021960 2024-11-24T13:47:01,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,967 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021960) roll requested 2024-11-24T13:47:01,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021967 2024-11-24T13:47:01,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021967 2024-11-24T13:47:01,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021967) roll requested 2024-11-24T13:47:01,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021979 2024-11-24T13:47:01,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021979 2024-11-24T13:47:01,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021979) roll requested 2024-11-24T13:47:01,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021985 2024-11-24T13:47:01,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021979 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021985 2024-11-24T13:47:01,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,990 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021985) roll requested 2024-11-24T13:47:01,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021991 2024-11-24T13:47:01,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:01,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021991 2024-11-24T13:47:01,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:01,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:01,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021991) roll requested 2024-11-24T13:47:01,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456021996 2024-11-24T13:47:02,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021996 2024-11-24T13:47:02,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,002 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456021996) roll requested 2024-11-24T13:47:02,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022002 2024-11-24T13:47:02,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456021996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022002 2024-11-24T13:47:02,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022002) roll requested 2024-11-24T13:47:02,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022008 2024-11-24T13:47:02,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022008 2024-11-24T13:47:02,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,015 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022008) roll requested 2024-11-24T13:47:02,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022015 2024-11-24T13:47:02,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022015 2024-11-24T13:47:02,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022015) roll requested 2024-11-24T13:47:02,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022024 2024-11-24T13:47:02,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022015 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022024 2024-11-24T13:47:02,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022024) roll requested 2024-11-24T13:47:02,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022032 2024-11-24T13:47:02,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022032 2024-11-24T13:47:02,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022032) roll requested 2024-11-24T13:47:02,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022041 2024-11-24T13:47:02,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022041 2024-11-24T13:47:02,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022041) roll requested 2024-11-24T13:47:02,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022048 2024-11-24T13:47:02,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022048 2024-11-24T13:47:02,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022048) roll requested 2024-11-24T13:47:02,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022056 2024-11-24T13:47:02,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022056 2024-11-24T13:47:02,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022056) roll requested 2024-11-24T13:47:02,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022067 2024-11-24T13:47:02,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022067 2024-11-24T13:47:02,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022067) roll requested 2024-11-24T13:47:02,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022075 2024-11-24T13:47:02,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022067 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022075 2024-11-24T13:47:02,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022075) roll requested 2024-11-24T13:47:02,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022080 2024-11-24T13:47:02,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022080 2024-11-24T13:47:02,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,091 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022080) roll requested 2024-11-24T13:47:02,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022092 2024-11-24T13:47:02,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022092 2024-11-24T13:47:02,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022092) roll requested 2024-11-24T13:47:02,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022098 2024-11-24T13:47:02,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022098 2024-11-24T13:47:02,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022098) roll requested 2024-11-24T13:47:02,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022108 2024-11-24T13:47:02,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022098 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022108 2024-11-24T13:47:02,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022108) roll requested 2024-11-24T13:47:02,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022113 2024-11-24T13:47:02,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022113 2024-11-24T13:47:02,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022113) roll requested 2024-11-24T13:47:02,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022119 2024-11-24T13:47:02,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022119 2024-11-24T13:47:02,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022119) roll requested 2024-11-24T13:47:02,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022127 2024-11-24T13:47:02,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022127 2024-11-24T13:47:02,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022127) roll requested 2024-11-24T13:47:02,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022135 2024-11-24T13:47:02,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022135 2024-11-24T13:47:02,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022135) roll requested 2024-11-24T13:47:02,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022142 2024-11-24T13:47:02,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022142 2024-11-24T13:47:02,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022142) roll requested 2024-11-24T13:47:02,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022165 2024-11-24T13:47:02,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022165 2024-11-24T13:47:02,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022165) roll requested 2024-11-24T13:47:02,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022179 2024-11-24T13:47:02,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022179 2024-11-24T13:47:02,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022179) roll requested 2024-11-24T13:47:02,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022185 2024-11-24T13:47:02,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022185 2024-11-24T13:47:02,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022185) roll requested 2024-11-24T13:47:02,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022195 2024-11-24T13:47:02,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022195 2024-11-24T13:47:02,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022195) roll requested 2024-11-24T13:47:02,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022207 2024-11-24T13:47:02,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022207 2024-11-24T13:47:02,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022207) roll requested 2024-11-24T13:47:02,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022219 2024-11-24T13:47:02,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022219 2024-11-24T13:47:02,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022219) roll requested 2024-11-24T13:47:02,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022226 2024-11-24T13:47:02,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022226 2024-11-24T13:47:02,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022226) roll requested 2024-11-24T13:47:02,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022232 2024-11-24T13:47:02,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022232 2024-11-24T13:47:02,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022232) roll requested 2024-11-24T13:47:02,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022240 2024-11-24T13:47:02,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022240 2024-11-24T13:47:02,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,250 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022240) roll requested 2024-11-24T13:47:02,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022250 2024-11-24T13:47:02,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022250 2024-11-24T13:47:02,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022250) roll requested 2024-11-24T13:47:02,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022257 2024-11-24T13:47:02,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022250 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022257 2024-11-24T13:47:02,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022257) roll requested 2024-11-24T13:47:02,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022267 2024-11-24T13:47:02,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022267 2024-11-24T13:47:02,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022267) roll requested 2024-11-24T13:47:02,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022274 2024-11-24T13:47:02,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022274 2024-11-24T13:47:02,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022274) roll requested 2024-11-24T13:47:02,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022282 2024-11-24T13:47:02,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022282 2024-11-24T13:47:02,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022282) roll requested 2024-11-24T13:47:02,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022289 2024-11-24T13:47:02,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022289 2024-11-24T13:47:02,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022289) roll requested 2024-11-24T13:47:02,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022300 2024-11-24T13:47:02,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022300 2024-11-24T13:47:02,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022300) roll requested 2024-11-24T13:47:02,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022327 2024-11-24T13:47:02,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022327 2024-11-24T13:47:02,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022327) roll requested 2024-11-24T13:47:02,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022346 2024-11-24T13:47:02,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022346 2024-11-24T13:47:02,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022346) roll requested 2024-11-24T13:47:02,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022354 2024-11-24T13:47:02,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022354 2024-11-24T13:47:02,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,364 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022354) roll requested 2024-11-24T13:47:02,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022364 2024-11-24T13:47:02,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022364 2024-11-24T13:47:02,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022364) roll requested 2024-11-24T13:47:02,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022370 2024-11-24T13:47:02,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022370 2024-11-24T13:47:02,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022370) roll requested 2024-11-24T13:47:02,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022376 2024-11-24T13:47:02,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022376 2024-11-24T13:47:02,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022376) roll requested 2024-11-24T13:47:02,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022382 2024-11-24T13:47:02,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022382 2024-11-24T13:47:02,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,388 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022382) roll requested 2024-11-24T13:47:02,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022389 2024-11-24T13:47:02,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022389 2024-11-24T13:47:02,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022389) roll requested 2024-11-24T13:47:02,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022395 2024-11-24T13:47:02,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022395 2024-11-24T13:47:02,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022395) roll requested 2024-11-24T13:47:02,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022403 2024-11-24T13:47:02,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022403 2024-11-24T13:47:02,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022403) roll requested 2024-11-24T13:47:02,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022413 2024-11-24T13:47:02,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022413 2024-11-24T13:47:02,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022413) roll requested 2024-11-24T13:47:02,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022421 2024-11-24T13:47:02,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022421 2024-11-24T13:47:02,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022421) roll requested 2024-11-24T13:47:02,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022429 2024-11-24T13:47:02,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022429 2024-11-24T13:47:02,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022429) roll requested 2024-11-24T13:47:02,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022434 2024-11-24T13:47:02,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022434 2024-11-24T13:47:02,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022434) roll requested 2024-11-24T13:47:02,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022442 2024-11-24T13:47:02,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022442 2024-11-24T13:47:02,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022442) roll requested 2024-11-24T13:47:02,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022449 2024-11-24T13:47:02,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022449 2024-11-24T13:47:02,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022449) roll requested 2024-11-24T13:47:02,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022456 2024-11-24T13:47:02,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022456 2024-11-24T13:47:02,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022456) roll requested 2024-11-24T13:47:02,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022464 2024-11-24T13:47:02,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022464 2024-11-24T13:47:02,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022464) roll requested 2024-11-24T13:47:02,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022473 2024-11-24T13:47:02,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022464 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022473 2024-11-24T13:47:02,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022473) roll requested 2024-11-24T13:47:02,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022480 2024-11-24T13:47:02,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022480 2024-11-24T13:47:02,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022480) roll requested 2024-11-24T13:47:02,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022489 2024-11-24T13:47:02,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022489 2024-11-24T13:47:02,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022489) roll requested 2024-11-24T13:47:02,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022498 2024-11-24T13:47:02,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022498 2024-11-24T13:47:02,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022498) roll requested 2024-11-24T13:47:02,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022505 2024-11-24T13:47:02,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022498 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022505 2024-11-24T13:47:02,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022505) roll requested 2024-11-24T13:47:02,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022514 2024-11-24T13:47:02,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022514 2024-11-24T13:47:02,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022514) roll requested 2024-11-24T13:47:02,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022521 2024-11-24T13:47:02,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022521 2024-11-24T13:47:02,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022521) roll requested 2024-11-24T13:47:02,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022528 2024-11-24T13:47:02,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022528 2024-11-24T13:47:02,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,538 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022528) roll requested 2024-11-24T13:47:02,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022538 2024-11-24T13:47:02,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022538 2024-11-24T13:47:02,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022538) roll requested 2024-11-24T13:47:02,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022545 2024-11-24T13:47:02,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022538 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022545 2024-11-24T13:47:02,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,554 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022545) roll requested 2024-11-24T13:47:02,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022555 2024-11-24T13:47:02,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022555 2024-11-24T13:47:02,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022555) roll requested 2024-11-24T13:47:02,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022571 2024-11-24T13:47:02,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022571 2024-11-24T13:47:02,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,579 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022571) roll requested 2024-11-24T13:47:02,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022579 2024-11-24T13:47:02,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022579 2024-11-24T13:47:02,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,586 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022579) roll requested 2024-11-24T13:47:02,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022587 2024-11-24T13:47:02,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022579 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022587 2024-11-24T13:47:02,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022587) roll requested 2024-11-24T13:47:02,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022593 2024-11-24T13:47:02,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022593 2024-11-24T13:47:02,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022593) roll requested 2024-11-24T13:47:02,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022599 2024-11-24T13:47:02,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022599 2024-11-24T13:47:02,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022599) roll requested 2024-11-24T13:47:02,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022611 2024-11-24T13:47:02,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022611 2024-11-24T13:47:02,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022611) roll requested 2024-11-24T13:47:02,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022620 2024-11-24T13:47:02,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022611 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022620 2024-11-24T13:47:02,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,626 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022620) roll requested 2024-11-24T13:47:02,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022627 2024-11-24T13:47:02,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022627 2024-11-24T13:47:02,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022627) roll requested 2024-11-24T13:47:02,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022632 2024-11-24T13:47:02,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022632 2024-11-24T13:47:02,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022632) roll requested 2024-11-24T13:47:02,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022639 2024-11-24T13:47:02,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022639 2024-11-24T13:47:02,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022639) roll requested 2024-11-24T13:47:02,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022648 2024-11-24T13:47:02,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022648 2024-11-24T13:47:02,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022648) roll requested 2024-11-24T13:47:02,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022663 2024-11-24T13:47:02,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022663 2024-11-24T13:47:02,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022663) roll requested 2024-11-24T13:47:02,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022675 2024-11-24T13:47:02,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022663 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022675 2024-11-24T13:47:02,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022675) roll requested 2024-11-24T13:47:02,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022681 2024-11-24T13:47:02,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022675 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022681 2024-11-24T13:47:02,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022681) roll requested 2024-11-24T13:47:02,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022689 2024-11-24T13:47:02,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022689 2024-11-24T13:47:02,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022689) roll requested 2024-11-24T13:47:02,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022694 2024-11-24T13:47:02,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022694 2024-11-24T13:47:02,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,699 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022694) roll requested 2024-11-24T13:47:02,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022699 2024-11-24T13:47:02,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022699 2024-11-24T13:47:02,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022699) roll requested 2024-11-24T13:47:02,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022705 2024-11-24T13:47:02,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022699 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022705 2024-11-24T13:47:02,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022705) roll requested 2024-11-24T13:47:02,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022711 2024-11-24T13:47:02,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022711 2024-11-24T13:47:02,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022711) roll requested 2024-11-24T13:47:02,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022718 2024-11-24T13:47:02,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022718 2024-11-24T13:47:02,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022718) roll requested 2024-11-24T13:47:02,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022726 2024-11-24T13:47:02,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022718 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022726 2024-11-24T13:47:02,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022726) roll requested 2024-11-24T13:47:02,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022734 2024-11-24T13:47:02,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022734 2024-11-24T13:47:02,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022734) roll requested 2024-11-24T13:47:02,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022742 2024-11-24T13:47:02,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022742 2024-11-24T13:47:02,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022742) roll requested 2024-11-24T13:47:02,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022749 2024-11-24T13:47:02,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022749 2024-11-24T13:47:02,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022749) roll requested 2024-11-24T13:47:02,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022757 2024-11-24T13:47:02,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022757 2024-11-24T13:47:02,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022757) roll requested 2024-11-24T13:47:02,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022765 2024-11-24T13:47:02,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022765 2024-11-24T13:47:02,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,774 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022765) roll requested 2024-11-24T13:47:02,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022775 2024-11-24T13:47:02,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022775 2024-11-24T13:47:02,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022775) roll requested 2024-11-24T13:47:02,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022781 2024-11-24T13:47:02,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022781 2024-11-24T13:47:02,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022781) roll requested 2024-11-24T13:47:02,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022789 2024-11-24T13:47:02,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022789 2024-11-24T13:47:02,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,798 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022789) roll requested 2024-11-24T13:47:02,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022799 2024-11-24T13:47:02,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022799 2024-11-24T13:47:02,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022799) roll requested 2024-11-24T13:47:02,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022806 2024-11-24T13:47:02,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022806 2024-11-24T13:47:02,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022806) roll requested 2024-11-24T13:47:02,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022813 2024-11-24T13:47:02,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022813 2024-11-24T13:47:02,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022813) roll requested 2024-11-24T13:47:02,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022818 2024-11-24T13:47:02,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022818 2024-11-24T13:47:02,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,823 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022818) roll requested 2024-11-24T13:47:02,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022824 2024-11-24T13:47:02,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022824 2024-11-24T13:47:02,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022824) roll requested 2024-11-24T13:47:02,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022829 2024-11-24T13:47:02,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022824 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022829 2024-11-24T13:47:02,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022829) roll requested 2024-11-24T13:47:02,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022835 2024-11-24T13:47:02,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022835 2024-11-24T13:47:02,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022835) roll requested 2024-11-24T13:47:02,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022841 2024-11-24T13:47:02,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022835 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022841 2024-11-24T13:47:02,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022841) roll requested 2024-11-24T13:47:02,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022850 2024-11-24T13:47:02,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022850 2024-11-24T13:47:02,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022850) roll requested 2024-11-24T13:47:02,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022858 2024-11-24T13:47:02,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022858 2024-11-24T13:47:02,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022858) roll requested 2024-11-24T13:47:02,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022862 2024-11-24T13:47:02,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022862 2024-11-24T13:47:02,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022862) roll requested 2024-11-24T13:47:02,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022867 2024-11-24T13:47:02,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022867 2024-11-24T13:47:02,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,874 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022867) roll requested 2024-11-24T13:47:02,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022874 2024-11-24T13:47:02,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022874 2024-11-24T13:47:02,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022874) roll requested 2024-11-24T13:47:02,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022882 2024-11-24T13:47:02,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022882 2024-11-24T13:47:02,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022882) roll requested 2024-11-24T13:47:02,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022890 2024-11-24T13:47:02,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022882 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022890 2024-11-24T13:47:02,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022890) roll requested 2024-11-24T13:47:02,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022900 2024-11-24T13:47:02,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022900 2024-11-24T13:47:02,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022900) roll requested 2024-11-24T13:47:02,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022906 2024-11-24T13:47:02,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022906 2024-11-24T13:47:02,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022906) roll requested 2024-11-24T13:47:02,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022911 2024-11-24T13:47:02,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022911 2024-11-24T13:47:02,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022911) roll requested 2024-11-24T13:47:02,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022917 2024-11-24T13:47:02,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022917 2024-11-24T13:47:02,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,923 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022917) roll requested 2024-11-24T13:47:02,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022924 2024-11-24T13:47:02,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022924 2024-11-24T13:47:02,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,930 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022924) roll requested 2024-11-24T13:47:02,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022930 2024-11-24T13:47:02,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022930 2024-11-24T13:47:02,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022930) roll requested 2024-11-24T13:47:02,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022939 2024-11-24T13:47:02,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022930 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022939 2024-11-24T13:47:02,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022939) roll requested 2024-11-24T13:47:02,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022945 2024-11-24T13:47:02,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022945 2024-11-24T13:47:02,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,950 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022945) roll requested 2024-11-24T13:47:02,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022951 2024-11-24T13:47:02,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022951 2024-11-24T13:47:02,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022951) roll requested 2024-11-24T13:47:02,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022957 2024-11-24T13:47:02,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022951 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022957 2024-11-24T13:47:02,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,963 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022957) roll requested 2024-11-24T13:47:02,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022963 2024-11-24T13:47:02,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022957 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022963 2024-11-24T13:47:02,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022963) roll requested 2024-11-24T13:47:02,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022969 2024-11-24T13:47:02,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022963 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022969 2024-11-24T13:47:02,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022969) roll requested 2024-11-24T13:47:02,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022976 2024-11-24T13:47:02,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022976 2024-11-24T13:47:02,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022976) roll requested 2024-11-24T13:47:02,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022985 2024-11-24T13:47:02,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:02,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022985 2024-11-24T13:47:02,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:02,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:02,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022985) roll requested 2024-11-24T13:47:02,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456022993 2024-11-24T13:47:03,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022993 2024-11-24T13:47:03,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,002 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456022993) roll requested 2024-11-24T13:47:03,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023003 2024-11-24T13:47:03,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456022993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023003 2024-11-24T13:47:03,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023003) roll requested 2024-11-24T13:47:03,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023009 2024-11-24T13:47:03,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023003 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023009 2024-11-24T13:47:03,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,015 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023009) roll requested 2024-11-24T13:47:03,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023015 2024-11-24T13:47:03,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023015 2024-11-24T13:47:03,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023015) roll requested 2024-11-24T13:47:03,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023026 2024-11-24T13:47:03,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023015 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023026 2024-11-24T13:47:03,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023026) roll requested 2024-11-24T13:47:03,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023031 2024-11-24T13:47:03,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023031 2024-11-24T13:47:03,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023031) roll requested 2024-11-24T13:47:03,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023036 2024-11-24T13:47:03,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023031 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023036 2024-11-24T13:47:03,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,042 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023036) roll requested 2024-11-24T13:47:03,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023043 2024-11-24T13:47:03,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023043 2024-11-24T13:47:03,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023043) roll requested 2024-11-24T13:47:03,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023049 2024-11-24T13:47:03,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023049 2024-11-24T13:47:03,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023049) roll requested 2024-11-24T13:47:03,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023057 2024-11-24T13:47:03,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023057 2024-11-24T13:47:03,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023057) roll requested 2024-11-24T13:47:03,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023069 2024-11-24T13:47:03,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023069 2024-11-24T13:47:03,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023069) roll requested 2024-11-24T13:47:03,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023075 2024-11-24T13:47:03,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023075 2024-11-24T13:47:03,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,081 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023075) roll requested 2024-11-24T13:47:03,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023081 2024-11-24T13:47:03,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023081 2024-11-24T13:47:03,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023081) roll requested 2024-11-24T13:47:03,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023087 2024-11-24T13:47:03,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023087 2024-11-24T13:47:03,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023087) roll requested 2024-11-24T13:47:03,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023092 2024-11-24T13:47:03,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023092 2024-11-24T13:47:03,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,097 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023092) roll requested 2024-11-24T13:47:03,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023097 2024-11-24T13:47:03,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023097 2024-11-24T13:47:03,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,101 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023097) roll requested 2024-11-24T13:47:03,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023102 2024-11-24T13:47:03,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023102 2024-11-24T13:47:03,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023102) roll requested 2024-11-24T13:47:03,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023107 2024-11-24T13:47:03,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023102 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023107 2024-11-24T13:47:03,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023107) roll requested 2024-11-24T13:47:03,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023118 2024-11-24T13:47:03,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023118 2024-11-24T13:47:03,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023118) roll requested 2024-11-24T13:47:03,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023124 2024-11-24T13:47:03,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023118 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023124 2024-11-24T13:47:03,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023124) roll requested 2024-11-24T13:47:03,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023129 2024-11-24T13:47:03,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023129 2024-11-24T13:47:03,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023129) roll requested 2024-11-24T13:47:03,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023134 2024-11-24T13:47:03,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023134 2024-11-24T13:47:03,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023134) roll requested 2024-11-24T13:47:03,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023140 2024-11-24T13:47:03,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023140 2024-11-24T13:47:03,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023140) roll requested 2024-11-24T13:47:03,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023145 2024-11-24T13:47:03,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023145 2024-11-24T13:47:03,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023145) roll requested 2024-11-24T13:47:03,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023151 2024-11-24T13:47:03,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023151 2024-11-24T13:47:03,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023151) roll requested 2024-11-24T13:47:03,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023164 2024-11-24T13:47:03,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023164 2024-11-24T13:47:03,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023164) roll requested 2024-11-24T13:47:03,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023170 2024-11-24T13:47:03,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023164 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023170 2024-11-24T13:47:03,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023170) roll requested 2024-11-24T13:47:03,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023179 2024-11-24T13:47:03,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023179 2024-11-24T13:47:03,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023179) roll requested 2024-11-24T13:47:03,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023186 2024-11-24T13:47:03,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023186 2024-11-24T13:47:03,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023186) roll requested 2024-11-24T13:47:03,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023199 2024-11-24T13:47:03,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023186 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023199 2024-11-24T13:47:03,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023199) roll requested 2024-11-24T13:47:03,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023219 2024-11-24T13:47:03,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023199 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023219 2024-11-24T13:47:03,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023219) roll requested 2024-11-24T13:47:03,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023230 2024-11-24T13:47:03,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023230 2024-11-24T13:47:03,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023230) roll requested 2024-11-24T13:47:03,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023238 2024-11-24T13:47:03,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023238 2024-11-24T13:47:03,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023238) roll requested 2024-11-24T13:47:03,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023244 2024-11-24T13:47:03,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023244 2024-11-24T13:47:03,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023244) roll requested 2024-11-24T13:47:03,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023249 2024-11-24T13:47:03,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023249 2024-11-24T13:47:03,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023249) roll requested 2024-11-24T13:47:03,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023255 2024-11-24T13:47:03,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023255 2024-11-24T13:47:03,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023255) roll requested 2024-11-24T13:47:03,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023263 2024-11-24T13:47:03,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023263 2024-11-24T13:47:03,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023263) roll requested 2024-11-24T13:47:03,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023270 2024-11-24T13:47:03,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023270 2024-11-24T13:47:03,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023270) roll requested 2024-11-24T13:47:03,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023285 2024-11-24T13:47:03,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023285 2024-11-24T13:47:03,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023285) roll requested 2024-11-24T13:47:03,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023292 2024-11-24T13:47:03,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023292 2024-11-24T13:47:03,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,302 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023292) roll requested 2024-11-24T13:47:03,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023302 2024-11-24T13:47:03,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023302 2024-11-24T13:47:03,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023302) roll requested 2024-11-24T13:47:03,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023310 2024-11-24T13:47:03,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023310 2024-11-24T13:47:03,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,315 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023310) roll requested 2024-11-24T13:47:03,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023315 2024-11-24T13:47:03,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023315 2024-11-24T13:47:03,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,323 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023315) roll requested 2024-11-24T13:47:03,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023323 2024-11-24T13:47:03,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023315 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023323 2024-11-24T13:47:03,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023323) roll requested 2024-11-24T13:47:03,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023328 2024-11-24T13:47:03,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023328 2024-11-24T13:47:03,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,333 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023328) roll requested 2024-11-24T13:47:03,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023333 2024-11-24T13:47:03,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023333 2024-11-24T13:47:03,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023333) roll requested 2024-11-24T13:47:03,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023338 2024-11-24T13:47:03,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023338 2024-11-24T13:47:03,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023338) roll requested 2024-11-24T13:47:03,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023346 2024-11-24T13:47:03,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023338 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023346 2024-11-24T13:47:03,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023346) roll requested 2024-11-24T13:47:03,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023353 2024-11-24T13:47:03,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023353 2024-11-24T13:47:03,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023353) roll requested 2024-11-24T13:47:03,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023362 2024-11-24T13:47:03,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023362 2024-11-24T13:47:03,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023362) roll requested 2024-11-24T13:47:03,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023372 2024-11-24T13:47:03,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023372 2024-11-24T13:47:03,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023372) roll requested 2024-11-24T13:47:03,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023379 2024-11-24T13:47:03,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023379 2024-11-24T13:47:03,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023379) roll requested 2024-11-24T13:47:03,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023387 2024-11-24T13:47:03,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023387 2024-11-24T13:47:03,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023387) roll requested 2024-11-24T13:47:03,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023396 2024-11-24T13:47:03,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023396 2024-11-24T13:47:03,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,401 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023396) roll requested 2024-11-24T13:47:03,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023401 2024-11-24T13:47:03,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023401 2024-11-24T13:47:03,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023401) roll requested 2024-11-24T13:47:03,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023406 2024-11-24T13:47:03,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023406 2024-11-24T13:47:03,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023406) roll requested 2024-11-24T13:47:03,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023422 2024-11-24T13:47:03,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023422 2024-11-24T13:47:03,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023422) roll requested 2024-11-24T13:47:03,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023431 2024-11-24T13:47:03,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023431 2024-11-24T13:47:03,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023431) roll requested 2024-11-24T13:47:03,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023439 2024-11-24T13:47:03,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023439 2024-11-24T13:47:03,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023439) roll requested 2024-11-24T13:47:03,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023448 2024-11-24T13:47:03,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023448 2024-11-24T13:47:03,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,462 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023448) roll requested 2024-11-24T13:47:03,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023463 2024-11-24T13:47:03,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023463 2024-11-24T13:47:03,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,470 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023463) roll requested 2024-11-24T13:47:03,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023471 2024-11-24T13:47:03,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023471 2024-11-24T13:47:03,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023471) roll requested 2024-11-24T13:47:03,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023479 2024-11-24T13:47:03,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023479 2024-11-24T13:47:03,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023479) roll requested 2024-11-24T13:47:03,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023484 2024-11-24T13:47:03,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023484 2024-11-24T13:47:03,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023484) roll requested 2024-11-24T13:47:03,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023489 2024-11-24T13:47:03,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023489 2024-11-24T13:47:03,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023489) roll requested 2024-11-24T13:47:03,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023494 2024-11-24T13:47:03,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023494 2024-11-24T13:47:03,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023494) roll requested 2024-11-24T13:47:03,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023503 2024-11-24T13:47:03,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023503 2024-11-24T13:47:03,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023503) roll requested 2024-11-24T13:47:03,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023508 2024-11-24T13:47:03,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023508 2024-11-24T13:47:03,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023508) roll requested 2024-11-24T13:47:03,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023514 2024-11-24T13:47:03,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023514 2024-11-24T13:47:03,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023514) roll requested 2024-11-24T13:47:03,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023519 2024-11-24T13:47:03,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023519 2024-11-24T13:47:03,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023519) roll requested 2024-11-24T13:47:03,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023524 2024-11-24T13:47:03,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023524 2024-11-24T13:47:03,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023524) roll requested 2024-11-24T13:47:03,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023530 2024-11-24T13:47:03,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023530 2024-11-24T13:47:03,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023530) roll requested 2024-11-24T13:47:03,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023535 2024-11-24T13:47:03,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023535 2024-11-24T13:47:03,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023535) roll requested 2024-11-24T13:47:03,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023541 2024-11-24T13:47:03,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023541 2024-11-24T13:47:03,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023541) roll requested 2024-11-24T13:47:03,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023547 2024-11-24T13:47:03,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023547 2024-11-24T13:47:03,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023547) roll requested 2024-11-24T13:47:03,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023552 2024-11-24T13:47:03,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023552 2024-11-24T13:47:03,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023552) roll requested 2024-11-24T13:47:03,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023558 2024-11-24T13:47:03,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023558 2024-11-24T13:47:03,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023558) roll requested 2024-11-24T13:47:03,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023564 2024-11-24T13:47:03,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023564 2024-11-24T13:47:03,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023564) roll requested 2024-11-24T13:47:03,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023577 2024-11-24T13:47:03,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023577 2024-11-24T13:47:03,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023577) roll requested 2024-11-24T13:47:03,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023584 2024-11-24T13:47:03,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023584 2024-11-24T13:47:03,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,590 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023584) roll requested 2024-11-24T13:47:03,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023590 2024-11-24T13:47:03,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023590 2024-11-24T13:47:03,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023590) roll requested 2024-11-24T13:47:03,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023597 2024-11-24T13:47:03,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023590 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023597 2024-11-24T13:47:03,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023597) roll requested 2024-11-24T13:47:03,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023606 2024-11-24T13:47:03,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023606 2024-11-24T13:47:03,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023606) roll requested 2024-11-24T13:47:03,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023615 2024-11-24T13:47:03,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023606 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023615 2024-11-24T13:47:03,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023615) roll requested 2024-11-24T13:47:03,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023627 2024-11-24T13:47:03,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023615 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023627 2024-11-24T13:47:03,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023627) roll requested 2024-11-24T13:47:03,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023634 2024-11-24T13:47:03,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023634 2024-11-24T13:47:03,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023634) roll requested 2024-11-24T13:47:03,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023642 2024-11-24T13:47:03,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023642 2024-11-24T13:47:03,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023642) roll requested 2024-11-24T13:47:03,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023647 2024-11-24T13:47:03,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023642 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023647 2024-11-24T13:47:03,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023647) roll requested 2024-11-24T13:47:03,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023652 2024-11-24T13:47:03,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023652 2024-11-24T13:47:03,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,658 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023652) roll requested 2024-11-24T13:47:03,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023659 2024-11-24T13:47:03,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023659 2024-11-24T13:47:03,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,666 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023659) roll requested 2024-11-24T13:47:03,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023666 2024-11-24T13:47:03,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023666 2024-11-24T13:47:03,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023666) roll requested 2024-11-24T13:47:03,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023674 2024-11-24T13:47:03,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023666 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023674 2024-11-24T13:47:03,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023674) roll requested 2024-11-24T13:47:03,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023683 2024-11-24T13:47:03,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023683 2024-11-24T13:47:03,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023683) roll requested 2024-11-24T13:47:03,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023689 2024-11-24T13:47:03,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023689 2024-11-24T13:47:03,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023689) roll requested 2024-11-24T13:47:03,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023696 2024-11-24T13:47:03,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023696 2024-11-24T13:47:03,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023696) roll requested 2024-11-24T13:47:03,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023704 2024-11-24T13:47:03,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023704 2024-11-24T13:47:03,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023704) roll requested 2024-11-24T13:47:03,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023710 2024-11-24T13:47:03,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023710 2024-11-24T13:47:03,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,722 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023710) roll requested 2024-11-24T13:47:03,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023722 2024-11-24T13:47:03,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023710 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023722 2024-11-24T13:47:03,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023722) roll requested 2024-11-24T13:47:03,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023729 2024-11-24T13:47:03,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023722 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023729 2024-11-24T13:47:03,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023729) roll requested 2024-11-24T13:47:03,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023748 2024-11-24T13:47:03,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023748 2024-11-24T13:47:03,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023748) roll requested 2024-11-24T13:47:03,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023758 2024-11-24T13:47:03,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023748 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023758 2024-11-24T13:47:03,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023758) roll requested 2024-11-24T13:47:03,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023764 2024-11-24T13:47:03,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023764 2024-11-24T13:47:03,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023764) roll requested 2024-11-24T13:47:03,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023771 2024-11-24T13:47:03,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023771 2024-11-24T13:47:03,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023771) roll requested 2024-11-24T13:47:03,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023783 2024-11-24T13:47:03,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023783 2024-11-24T13:47:03,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023783) roll requested 2024-11-24T13:47:03,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023789 2024-11-24T13:47:03,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023789 2024-11-24T13:47:03,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023789) roll requested 2024-11-24T13:47:03,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023796 2024-11-24T13:47:03,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023796 2024-11-24T13:47:03,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023796) roll requested 2024-11-24T13:47:03,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023802 2024-11-24T13:47:03,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023802 2024-11-24T13:47:03,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023802) roll requested 2024-11-24T13:47:03,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023810 2024-11-24T13:47:03,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023810 2024-11-24T13:47:03,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023810) roll requested 2024-11-24T13:47:03,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023815 2024-11-24T13:47:03,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023810 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023815 2024-11-24T13:47:03,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023815) roll requested 2024-11-24T13:47:03,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023822 2024-11-24T13:47:03,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023822 2024-11-24T13:47:03,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,830 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023822) roll requested 2024-11-24T13:47:03,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023830 2024-11-24T13:47:03,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023830 2024-11-24T13:47:03,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023830) roll requested 2024-11-24T13:47:03,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023841 2024-11-24T13:47:03,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023830 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023841 2024-11-24T13:47:03,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023841) roll requested 2024-11-24T13:47:03,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023847 2024-11-24T13:47:03,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023847 2024-11-24T13:47:03,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023847) roll requested 2024-11-24T13:47:03,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023853 2024-11-24T13:47:03,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023853 2024-11-24T13:47:03,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023853) roll requested 2024-11-24T13:47:03,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023861 2024-11-24T13:47:03,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023861 2024-11-24T13:47:03,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,874 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023861) roll requested 2024-11-24T13:47:03,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023875 2024-11-24T13:47:03,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023875 2024-11-24T13:47:03,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023875) roll requested 2024-11-24T13:47:03,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023883 2024-11-24T13:47:03,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023875 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023883 2024-11-24T13:47:03,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,891 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023883) roll requested 2024-11-24T13:47:03,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023891 2024-11-24T13:47:03,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023891 2024-11-24T13:47:03,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023891) roll requested 2024-11-24T13:47:03,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023897 2024-11-24T13:47:03,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023897 2024-11-24T13:47:03,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023897) roll requested 2024-11-24T13:47:03,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023905 2024-11-24T13:47:03,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023897 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023905 2024-11-24T13:47:03,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023905) roll requested 2024-11-24T13:47:03,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023911 2024-11-24T13:47:03,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023911 2024-11-24T13:47:03,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023911) roll requested 2024-11-24T13:47:03,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023916 2024-11-24T13:47:03,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023916 2024-11-24T13:47:03,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,922 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023916) roll requested 2024-11-24T13:47:03,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023922 2024-11-24T13:47:03,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023922 2024-11-24T13:47:03,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,930 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023922) roll requested 2024-11-24T13:47:03,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023930 2024-11-24T13:47:03,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023930 2024-11-24T13:47:03,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023930) roll requested 2024-11-24T13:47:03,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023936 2024-11-24T13:47:03,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023930 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023936 2024-11-24T13:47:03,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023936) roll requested 2024-11-24T13:47:03,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023947 2024-11-24T13:47:03,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023947 2024-11-24T13:47:03,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023947) roll requested 2024-11-24T13:47:03,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023955 2024-11-24T13:47:03,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023955 2024-11-24T13:47:03,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,963 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023955) roll requested 2024-11-24T13:47:03,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023963 2024-11-24T13:47:03,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023955 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023963 2024-11-24T13:47:03,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023963) roll requested 2024-11-24T13:47:03,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023969 2024-11-24T13:47:03,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023963 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023969 2024-11-24T13:47:03,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,974 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023969) roll requested 2024-11-24T13:47:03,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023974 2024-11-24T13:47:03,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023974 2024-11-24T13:47:03,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023974) roll requested 2024-11-24T13:47:03,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023981 2024-11-24T13:47:03,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023981 2024-11-24T13:47:03,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,990 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023981) roll requested 2024-11-24T13:47:03,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023990 2024-11-24T13:47:03,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:03,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023990 2024-11-24T13:47:03,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:03,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:03,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023990) roll requested 2024-11-24T13:47:03,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456023995 2024-11-24T13:47:04,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023990 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023995 2024-11-24T13:47:04,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,002 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456023995) roll requested 2024-11-24T13:47:04,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024002 2024-11-24T13:47:04,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456023995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024002 2024-11-24T13:47:04,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024002) roll requested 2024-11-24T13:47:04,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024011 2024-11-24T13:47:04,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024011 2024-11-24T13:47:04,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024011) roll requested 2024-11-24T13:47:04,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024031 2024-11-24T13:47:04,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024031 2024-11-24T13:47:04,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024031) roll requested 2024-11-24T13:47:04,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024040 2024-11-24T13:47:04,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024031 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024040 2024-11-24T13:47:04,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024040) roll requested 2024-11-24T13:47:04,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024051 2024-11-24T13:47:04,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024051 2024-11-24T13:47:04,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024051) roll requested 2024-11-24T13:47:04,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024063 2024-11-24T13:47:04,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024063 2024-11-24T13:47:04,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024063) roll requested 2024-11-24T13:47:04,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024069 2024-11-24T13:47:04,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024063 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024069 2024-11-24T13:47:04,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024069) roll requested 2024-11-24T13:47:04,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024083 2024-11-24T13:47:04,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024083 2024-11-24T13:47:04,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024083) roll requested 2024-11-24T13:47:04,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024094 2024-11-24T13:47:04,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024094 2024-11-24T13:47:04,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024094) roll requested 2024-11-24T13:47:04,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024103 2024-11-24T13:47:04,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024103 2024-11-24T13:47:04,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024103) roll requested 2024-11-24T13:47:04,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024109 2024-11-24T13:47:04,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024109 2024-11-24T13:47:04,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,117 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024109) roll requested 2024-11-24T13:47:04,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024117 2024-11-24T13:47:04,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024117 2024-11-24T13:47:04,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024117) roll requested 2024-11-24T13:47:04,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024122 2024-11-24T13:47:04,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024122 2024-11-24T13:47:04,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024122) roll requested 2024-11-24T13:47:04,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024135 2024-11-24T13:47:04,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024135 2024-11-24T13:47:04,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024135) roll requested 2024-11-24T13:47:04,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024141 2024-11-24T13:47:04,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024141 2024-11-24T13:47:04,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024141) roll requested 2024-11-24T13:47:04,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024147 2024-11-24T13:47:04,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024147 2024-11-24T13:47:04,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,155 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024147) roll requested 2024-11-24T13:47:04,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024155 2024-11-24T13:47:04,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024147 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024155 2024-11-24T13:47:04,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024155) roll requested 2024-11-24T13:47:04,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024161 2024-11-24T13:47:04,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024155 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024161 2024-11-24T13:47:04,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,166 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024161) roll requested 2024-11-24T13:47:04,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024166 2024-11-24T13:47:04,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024166 2024-11-24T13:47:04,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024166) roll requested 2024-11-24T13:47:04,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024171 2024-11-24T13:47:04,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024166 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024171 2024-11-24T13:47:04,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024171) roll requested 2024-11-24T13:47:04,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024177 2024-11-24T13:47:04,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024177 2024-11-24T13:47:04,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024177) roll requested 2024-11-24T13:47:04,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024182 2024-11-24T13:47:04,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024182 2024-11-24T13:47:04,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,187 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024182) roll requested 2024-11-24T13:47:04,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024187 2024-11-24T13:47:04,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024182 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024187 2024-11-24T13:47:04,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,193 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024187) roll requested 2024-11-24T13:47:04,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024193 2024-11-24T13:47:04,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024193 2024-11-24T13:47:04,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024193) roll requested 2024-11-24T13:47:04,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024199 2024-11-24T13:47:04,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024199 2024-11-24T13:47:04,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024199) roll requested 2024-11-24T13:47:04,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024207 2024-11-24T13:47:04,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024199 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024207 2024-11-24T13:47:04,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024207) roll requested 2024-11-24T13:47:04,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024212 2024-11-24T13:47:04,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024212 2024-11-24T13:47:04,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024212) roll requested 2024-11-24T13:47:04,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024218 2024-11-24T13:47:04,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024218 2024-11-24T13:47:04,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024218) roll requested 2024-11-24T13:47:04,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024224 2024-11-24T13:47:04,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024218 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024224 2024-11-24T13:47:04,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024224) roll requested 2024-11-24T13:47:04,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024229 2024-11-24T13:47:04,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024229 2024-11-24T13:47:04,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,234 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024229) roll requested 2024-11-24T13:47:04,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024234 2024-11-24T13:47:04,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024234 2024-11-24T13:47:04,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024234) roll requested 2024-11-24T13:47:04,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024240 2024-11-24T13:47:04,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024240 2024-11-24T13:47:04,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024240) roll requested 2024-11-24T13:47:04,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024245 2024-11-24T13:47:04,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024245 2024-11-24T13:47:04,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024245) roll requested 2024-11-24T13:47:04,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024251 2024-11-24T13:47:04,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024251 2024-11-24T13:47:04,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024251) roll requested 2024-11-24T13:47:04,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024260 2024-11-24T13:47:04,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024260 2024-11-24T13:47:04,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024260) roll requested 2024-11-24T13:47:04,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024270 2024-11-24T13:47:04,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024270 2024-11-24T13:47:04,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024270) roll requested 2024-11-24T13:47:04,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024277 2024-11-24T13:47:04,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024277 2024-11-24T13:47:04,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,282 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024277) roll requested 2024-11-24T13:47:04,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024282 2024-11-24T13:47:04,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024282 2024-11-24T13:47:04,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024282) roll requested 2024-11-24T13:47:04,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024287 2024-11-24T13:47:04,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024287 2024-11-24T13:47:04,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024287) roll requested 2024-11-24T13:47:04,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024293 2024-11-24T13:47:04,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024293 2024-11-24T13:47:04,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024293) roll requested 2024-11-24T13:47:04,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024298 2024-11-24T13:47:04,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024298 2024-11-24T13:47:04,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,303 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024298) roll requested 2024-11-24T13:47:04,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024303 2024-11-24T13:47:04,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024303 2024-11-24T13:47:04,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024303) roll requested 2024-11-24T13:47:04,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024309 2024-11-24T13:47:04,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024303 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024309 2024-11-24T13:47:04,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024309) roll requested 2024-11-24T13:47:04,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024314 2024-11-24T13:47:04,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024314 2024-11-24T13:47:04,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024314) roll requested 2024-11-24T13:47:04,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024319 2024-11-24T13:47:04,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024319 2024-11-24T13:47:04,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024319) roll requested 2024-11-24T13:47:04,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024325 2024-11-24T13:47:04,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024325 2024-11-24T13:47:04,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024325) roll requested 2024-11-24T13:47:04,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024330 2024-11-24T13:47:04,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024330 2024-11-24T13:47:04,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024330) roll requested 2024-11-24T13:47:04,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024339 2024-11-24T13:47:04,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024330 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024339 2024-11-24T13:47:04,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024339) roll requested 2024-11-24T13:47:04,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024349 2024-11-24T13:47:04,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024349 2024-11-24T13:47:04,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024349) roll requested 2024-11-24T13:47:04,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024370 2024-11-24T13:47:04,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024370 2024-11-24T13:47:04,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024370) roll requested 2024-11-24T13:47:04,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024379 2024-11-24T13:47:04,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024379 2024-11-24T13:47:04,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024379) roll requested 2024-11-24T13:47:04,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024387 2024-11-24T13:47:04,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024387 2024-11-24T13:47:04,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,393 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024387) roll requested 2024-11-24T13:47:04,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024394 2024-11-24T13:47:04,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024394 2024-11-24T13:47:04,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024394) roll requested 2024-11-24T13:47:04,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024405 2024-11-24T13:47:04,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024405 2024-11-24T13:47:04,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024405) roll requested 2024-11-24T13:47:04,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024415 2024-11-24T13:47:04,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024415 2024-11-24T13:47:04,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024415) roll requested 2024-11-24T13:47:04,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024423 2024-11-24T13:47:04,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024423 2024-11-24T13:47:04,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024423) roll requested 2024-11-24T13:47:04,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024432 2024-11-24T13:47:04,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024423 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024432 2024-11-24T13:47:04,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024432) roll requested 2024-11-24T13:47:04,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024439 2024-11-24T13:47:04,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024439 2024-11-24T13:47:04,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024439) roll requested 2024-11-24T13:47:04,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024447 2024-11-24T13:47:04,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024447 2024-11-24T13:47:04,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,454 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024447) roll requested 2024-11-24T13:47:04,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024454 2024-11-24T13:47:04,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024454 2024-11-24T13:47:04,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024454) roll requested 2024-11-24T13:47:04,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024459 2024-11-24T13:47:04,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024459 2024-11-24T13:47:04,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024459) roll requested 2024-11-24T13:47:04,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024464 2024-11-24T13:47:04,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024464 2024-11-24T13:47:04,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024464) roll requested 2024-11-24T13:47:04,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024469 2024-11-24T13:47:04,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024464 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024469 2024-11-24T13:47:04,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024469) roll requested 2024-11-24T13:47:04,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024474 2024-11-24T13:47:04,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024474 2024-11-24T13:47:04,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024474) roll requested 2024-11-24T13:47:04,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024478 2024-11-24T13:47:04,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024478 2024-11-24T13:47:04,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024478) roll requested 2024-11-24T13:47:04,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024483 2024-11-24T13:47:04,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024483 2024-11-24T13:47:04,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024483) roll requested 2024-11-24T13:47:04,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024491 2024-11-24T13:47:04,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024491 2024-11-24T13:47:04,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024491) roll requested 2024-11-24T13:47:04,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024496 2024-11-24T13:47:04,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024496 2024-11-24T13:47:04,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024496) roll requested 2024-11-24T13:47:04,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024501 2024-11-24T13:47:04,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024501 2024-11-24T13:47:04,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024501) roll requested 2024-11-24T13:47:04,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024506 2024-11-24T13:47:04,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024506 2024-11-24T13:47:04,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024506) roll requested 2024-11-24T13:47:04,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024514 2024-11-24T13:47:04,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024514 2024-11-24T13:47:04,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024514) roll requested 2024-11-24T13:47:04,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024527 2024-11-24T13:47:04,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024527 2024-11-24T13:47:04,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024527) roll requested 2024-11-24T13:47:04,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024541 2024-11-24T13:47:04,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024541 2024-11-24T13:47:04,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024541) roll requested 2024-11-24T13:47:04,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024550 2024-11-24T13:47:04,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024550 2024-11-24T13:47:04,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024550) roll requested 2024-11-24T13:47:04,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024557 2024-11-24T13:47:04,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024550 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024557 2024-11-24T13:47:04,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024557) roll requested 2024-11-24T13:47:04,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024562 2024-11-24T13:47:04,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024562 2024-11-24T13:47:04,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,567 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024562) roll requested 2024-11-24T13:47:04,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024568 2024-11-24T13:47:04,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024562 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024568 2024-11-24T13:47:04,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024568) roll requested 2024-11-24T13:47:04,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024573 2024-11-24T13:47:04,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024573 2024-11-24T13:47:04,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024573) roll requested 2024-11-24T13:47:04,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024578 2024-11-24T13:47:04,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024573 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024578 2024-11-24T13:47:04,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024578) roll requested 2024-11-24T13:47:04,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024585 2024-11-24T13:47:04,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024585 2024-11-24T13:47:04,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024585) roll requested 2024-11-24T13:47:04,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024592 2024-11-24T13:47:04,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024592 2024-11-24T13:47:04,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024592) roll requested 2024-11-24T13:47:04,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024600 2024-11-24T13:47:04,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024600 2024-11-24T13:47:04,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024600) roll requested 2024-11-24T13:47:04,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024608 2024-11-24T13:47:04,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024600 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024608 2024-11-24T13:47:04,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024608) roll requested 2024-11-24T13:47:04,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024618 2024-11-24T13:47:04,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024618 2024-11-24T13:47:04,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024618) roll requested 2024-11-24T13:47:04,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024627 2024-11-24T13:47:04,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024627 2024-11-24T13:47:04,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024627) roll requested 2024-11-24T13:47:04,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024636 2024-11-24T13:47:04,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024636 2024-11-24T13:47:04,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024636) roll requested 2024-11-24T13:47:04,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024646 2024-11-24T13:47:04,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024646 2024-11-24T13:47:04,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024646) roll requested 2024-11-24T13:47:04,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024655 2024-11-24T13:47:04,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024646 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024655 2024-11-24T13:47:04,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024655) roll requested 2024-11-24T13:47:04,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024660 2024-11-24T13:47:04,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024660 2024-11-24T13:47:04,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,666 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024660) roll requested 2024-11-24T13:47:04,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024666 2024-11-24T13:47:04,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024666 2024-11-24T13:47:04,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,673 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024666) roll requested 2024-11-24T13:47:04,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024674 2024-11-24T13:47:04,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024666 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024674 2024-11-24T13:47:04,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024674) roll requested 2024-11-24T13:47:04,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024682 2024-11-24T13:47:04,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024682 2024-11-24T13:47:04,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024682) roll requested 2024-11-24T13:47:04,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024689 2024-11-24T13:47:04,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024689 2024-11-24T13:47:04,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,698 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024689) roll requested 2024-11-24T13:47:04,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024698 2024-11-24T13:47:04,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024698 2024-11-24T13:47:04,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024698) roll requested 2024-11-24T13:47:04,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024708 2024-11-24T13:47:04,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024708 2024-11-24T13:47:04,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024708) roll requested 2024-11-24T13:47:04,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024716 2024-11-24T13:47:04,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024716 2024-11-24T13:47:04,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,722 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024716) roll requested 2024-11-24T13:47:04,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024722 2024-11-24T13:47:04,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024716 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024722 2024-11-24T13:47:04,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,730 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024722) roll requested 2024-11-24T13:47:04,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024730 2024-11-24T13:47:04,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024722 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024730 2024-11-24T13:47:04,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,741 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024730) roll requested 2024-11-24T13:47:04,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024742 2024-11-24T13:47:04,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024742 2024-11-24T13:47:04,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024742) roll requested 2024-11-24T13:47:04,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024751 2024-11-24T13:47:04,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024751 2024-11-24T13:47:04,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,762 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024751) roll requested 2024-11-24T13:47:04,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024762 2024-11-24T13:47:04,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024762 2024-11-24T13:47:04,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024762) roll requested 2024-11-24T13:47:04,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024769 2024-11-24T13:47:04,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024762 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024769 2024-11-24T13:47:04,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,777 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024769) roll requested 2024-11-24T13:47:04,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024777 2024-11-24T13:47:04,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024777 2024-11-24T13:47:04,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024777) roll requested 2024-11-24T13:47:04,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024784 2024-11-24T13:47:04,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024784 2024-11-24T13:47:04,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024784) roll requested 2024-11-24T13:47:04,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024789 2024-11-24T13:47:04,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024789 2024-11-24T13:47:04,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,798 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024789) roll requested 2024-11-24T13:47:04,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024798 2024-11-24T13:47:04,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024798 2024-11-24T13:47:04,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024798) roll requested 2024-11-24T13:47:04,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024804 2024-11-24T13:47:04,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024798 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024804 2024-11-24T13:47:04,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024804) roll requested 2024-11-24T13:47:04,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024808 2024-11-24T13:47:04,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024808 2024-11-24T13:47:04,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024808) roll requested 2024-11-24T13:47:04,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024815 2024-11-24T13:47:04,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024815 2024-11-24T13:47:04,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024815) roll requested 2024-11-24T13:47:04,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024820 2024-11-24T13:47:04,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024820 2024-11-24T13:47:04,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024820) roll requested 2024-11-24T13:47:04,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024825 2024-11-24T13:47:04,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024825 2024-11-24T13:47:04,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,830 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024825) roll requested 2024-11-24T13:47:04,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024830 2024-11-24T13:47:04,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024830 2024-11-24T13:47:04,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024830) roll requested 2024-11-24T13:47:04,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024835 2024-11-24T13:47:04,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024830 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024835 2024-11-24T13:47:04,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024835) roll requested 2024-11-24T13:47:04,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024840 2024-11-24T13:47:04,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024835 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024840 2024-11-24T13:47:04,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024840) roll requested 2024-11-24T13:47:04,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024845 2024-11-24T13:47:04,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024845 2024-11-24T13:47:04,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024845) roll requested 2024-11-24T13:47:04,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024850 2024-11-24T13:47:04,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024850 2024-11-24T13:47:04,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024850) roll requested 2024-11-24T13:47:04,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024855 2024-11-24T13:47:04,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024855 2024-11-24T13:47:04,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024855) roll requested 2024-11-24T13:47:04,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024865 2024-11-24T13:47:04,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024865 2024-11-24T13:47:04,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024865) roll requested 2024-11-24T13:47:04,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024878 2024-11-24T13:47:04,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024878 2024-11-24T13:47:04,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024878) roll requested 2024-11-24T13:47:04,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024890 2024-11-24T13:47:04,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024890 2024-11-24T13:47:04,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,903 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024890) roll requested 2024-11-24T13:47:04,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024903 2024-11-24T13:47:04,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024903 2024-11-24T13:47:04,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,909 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024903) roll requested 2024-11-24T13:47:04,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024909 2024-11-24T13:47:04,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024909 2024-11-24T13:47:04,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,915 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024909) roll requested 2024-11-24T13:47:04,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024915 2024-11-24T13:47:04,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024915 2024-11-24T13:47:04,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024915) roll requested 2024-11-24T13:47:04,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024920 2024-11-24T13:47:04,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024920 2024-11-24T13:47:04,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024920) roll requested 2024-11-24T13:47:04,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024927 2024-11-24T13:47:04,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024927 2024-11-24T13:47:04,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024927) roll requested 2024-11-24T13:47:04,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024933 2024-11-24T13:47:04,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024933 2024-11-24T13:47:04,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024933) roll requested 2024-11-24T13:47:04,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024939 2024-11-24T13:47:04,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024939 2024-11-24T13:47:04,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024939) roll requested 2024-11-24T13:47:04,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024944 2024-11-24T13:47:04,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024944 2024-11-24T13:47:04,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024944) roll requested 2024-11-24T13:47:04,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024949 2024-11-24T13:47:04,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024949 2024-11-24T13:47:04,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024949) roll requested 2024-11-24T13:47:04,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024954 2024-11-24T13:47:04,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024954 2024-11-24T13:47:04,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024954) roll requested 2024-11-24T13:47:04,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024961 2024-11-24T13:47:04,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024961 2024-11-24T13:47:04,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024961) roll requested 2024-11-24T13:47:04,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024967 2024-11-24T13:47:04,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024967 2024-11-24T13:47:04,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024967) roll requested 2024-11-24T13:47:04,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024972 2024-11-24T13:47:04,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024972 2024-11-24T13:47:04,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024972) roll requested 2024-11-24T13:47:04,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024977 2024-11-24T13:47:04,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024977 2024-11-24T13:47:04,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024977) roll requested 2024-11-24T13:47:04,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024982 2024-11-24T13:47:04,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024982 2024-11-24T13:47:04,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024982) roll requested 2024-11-24T13:47:04,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024987 2024-11-24T13:47:04,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024987 2024-11-24T13:47:04,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024987) roll requested 2024-11-24T13:47:04,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024992 2024-11-24T13:47:04,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:04,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024992 2024-11-24T13:47:04,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:04,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:04,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024992) roll requested 2024-11-24T13:47:04,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456024997 2024-11-24T13:47:05,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024997 2024-11-24T13:47:05,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456024997) roll requested 2024-11-24T13:47:05,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025002 2024-11-24T13:47:05,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456024997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025002 2024-11-24T13:47:05,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025002) roll requested 2024-11-24T13:47:05,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025007 2024-11-24T13:47:05,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025007 2024-11-24T13:47:05,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025007) roll requested 2024-11-24T13:47:05,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025014 2024-11-24T13:47:05,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025014 2024-11-24T13:47:05,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,024 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025014) roll requested 2024-11-24T13:47:05,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025025 2024-11-24T13:47:05,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025025 2024-11-24T13:47:05,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025025) roll requested 2024-11-24T13:47:05,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025034 2024-11-24T13:47:05,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025034 2024-11-24T13:47:05,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025034) roll requested 2024-11-24T13:47:05,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025044 2024-11-24T13:47:05,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025044 2024-11-24T13:47:05,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,050 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025044) roll requested 2024-11-24T13:47:05,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025051 2024-11-24T13:47:05,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025051 2024-11-24T13:47:05,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025051) roll requested 2024-11-24T13:47:05,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025056 2024-11-24T13:47:05,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025056 2024-11-24T13:47:05,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025056) roll requested 2024-11-24T13:47:05,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025063 2024-11-24T13:47:05,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025063 2024-11-24T13:47:05,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025063) roll requested 2024-11-24T13:47:05,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025069 2024-11-24T13:47:05,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025063 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025069 2024-11-24T13:47:05,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025069) roll requested 2024-11-24T13:47:05,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025089 2024-11-24T13:47:05,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025089 2024-11-24T13:47:05,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025089) roll requested 2024-11-24T13:47:05,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025096 2024-11-24T13:47:05,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025096 2024-11-24T13:47:05,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025096) roll requested 2024-11-24T13:47:05,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025106 2024-11-24T13:47:05,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025106 2024-11-24T13:47:05,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025106) roll requested 2024-11-24T13:47:05,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025115 2024-11-24T13:47:05,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025115 2024-11-24T13:47:05,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025115) roll requested 2024-11-24T13:47:05,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025121 2024-11-24T13:47:05,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025115 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025121 2024-11-24T13:47:05,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,126 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025121) roll requested 2024-11-24T13:47:05,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025127 2024-11-24T13:47:05,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025127 2024-11-24T13:47:05,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025127) roll requested 2024-11-24T13:47:05,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025136 2024-11-24T13:47:05,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025136 2024-11-24T13:47:05,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025136) roll requested 2024-11-24T13:47:05,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025142 2024-11-24T13:47:05,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025142 2024-11-24T13:47:05,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,146 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025142) roll requested 2024-11-24T13:47:05,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025147 2024-11-24T13:47:05,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025147 2024-11-24T13:47:05,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025147) roll requested 2024-11-24T13:47:05,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025152 2024-11-24T13:47:05,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025147 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025152 2024-11-24T13:47:05,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025152) roll requested 2024-11-24T13:47:05,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025157 2024-11-24T13:47:05,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025157 2024-11-24T13:47:05,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,163 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025157) roll requested 2024-11-24T13:47:05,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025164 2024-11-24T13:47:05,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025164 2024-11-24T13:47:05,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025164) roll requested 2024-11-24T13:47:05,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025171 2024-11-24T13:47:05,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025164 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025171 2024-11-24T13:47:05,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025171) roll requested 2024-11-24T13:47:05,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025179 2024-11-24T13:47:05,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025179 2024-11-24T13:47:05,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025179) roll requested 2024-11-24T13:47:05,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025186 2024-11-24T13:47:05,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025186 2024-11-24T13:47:05,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025186) roll requested 2024-11-24T13:47:05,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025195 2024-11-24T13:47:05,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025186 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025195 2024-11-24T13:47:05,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025195) roll requested 2024-11-24T13:47:05,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025206 2024-11-24T13:47:05,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025206 2024-11-24T13:47:05,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025206) roll requested 2024-11-24T13:47:05,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025215 2024-11-24T13:47:05,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025215 2024-11-24T13:47:05,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025215) roll requested 2024-11-24T13:47:05,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025223 2024-11-24T13:47:05,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025223 2024-11-24T13:47:05,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025223) roll requested 2024-11-24T13:47:05,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025230 2024-11-24T13:47:05,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025230 2024-11-24T13:47:05,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025230) roll requested 2024-11-24T13:47:05,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025236 2024-11-24T13:47:05,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025236 2024-11-24T13:47:05,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025236) roll requested 2024-11-24T13:47:05,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025247 2024-11-24T13:47:05,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025236 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025247 2024-11-24T13:47:05,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025247) roll requested 2024-11-24T13:47:05,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025253 2024-11-24T13:47:05,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025253 2024-11-24T13:47:05,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025253) roll requested 2024-11-24T13:47:05,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025259 2024-11-24T13:47:05,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025259 2024-11-24T13:47:05,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025259) roll requested 2024-11-24T13:47:05,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025265 2024-11-24T13:47:05,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025265 2024-11-24T13:47:05,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025265) roll requested 2024-11-24T13:47:05,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025270 2024-11-24T13:47:05,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025270 2024-11-24T13:47:05,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025270) roll requested 2024-11-24T13:47:05,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025280 2024-11-24T13:47:05,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025280 2024-11-24T13:47:05,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025280) roll requested 2024-11-24T13:47:05,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025286 2024-11-24T13:47:05,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025286 2024-11-24T13:47:05,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025286) roll requested 2024-11-24T13:47:05,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025295 2024-11-24T13:47:05,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025295 2024-11-24T13:47:05,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025295) roll requested 2024-11-24T13:47:05,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025301 2024-11-24T13:47:05,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025301 2024-11-24T13:47:05,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025301) roll requested 2024-11-24T13:47:05,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025307 2024-11-24T13:47:05,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025307 2024-11-24T13:47:05,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025307) roll requested 2024-11-24T13:47:05,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025313 2024-11-24T13:47:05,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025313 2024-11-24T13:47:05,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025313) roll requested 2024-11-24T13:47:05,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025320 2024-11-24T13:47:05,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025320 2024-11-24T13:47:05,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025320) roll requested 2024-11-24T13:47:05,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025327 2024-11-24T13:47:05,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025327 2024-11-24T13:47:05,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025327) roll requested 2024-11-24T13:47:05,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025335 2024-11-24T13:47:05,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025335 2024-11-24T13:47:05,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025335) roll requested 2024-11-24T13:47:05,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025342 2024-11-24T13:47:05,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025342 2024-11-24T13:47:05,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025342) roll requested 2024-11-24T13:47:05,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025347 2024-11-24T13:47:05,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025342 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025347 2024-11-24T13:47:05,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025347) roll requested 2024-11-24T13:47:05,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025352 2024-11-24T13:47:05,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025352 2024-11-24T13:47:05,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,358 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025352) roll requested 2024-11-24T13:47:05,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025358 2024-11-24T13:47:05,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025358 2024-11-24T13:47:05,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,364 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025358) roll requested 2024-11-24T13:47:05,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025364 2024-11-24T13:47:05,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025358 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025364 2024-11-24T13:47:05,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025364) roll requested 2024-11-24T13:47:05,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025369 2024-11-24T13:47:05,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025369 2024-11-24T13:47:05,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025369) roll requested 2024-11-24T13:47:05,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025375 2024-11-24T13:47:05,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025375 2024-11-24T13:47:05,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025375) roll requested 2024-11-24T13:47:05,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025380 2024-11-24T13:47:05,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025380 2024-11-24T13:47:05,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,385 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025380) roll requested 2024-11-24T13:47:05,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025385 2024-11-24T13:47:05,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025385 2024-11-24T13:47:05,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,390 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025385) roll requested 2024-11-24T13:47:05,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025390 2024-11-24T13:47:05,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025390 2024-11-24T13:47:05,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025390) roll requested 2024-11-24T13:47:05,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025395 2024-11-24T13:47:05,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025395 2024-11-24T13:47:05,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025395) roll requested 2024-11-24T13:47:05,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025400 2024-11-24T13:47:05,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025400 2024-11-24T13:47:05,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025400) roll requested 2024-11-24T13:47:05,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025407 2024-11-24T13:47:05,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025407 2024-11-24T13:47:05,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025407) roll requested 2024-11-24T13:47:05,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025413 2024-11-24T13:47:05,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025413 2024-11-24T13:47:05,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025413) roll requested 2024-11-24T13:47:05,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025421 2024-11-24T13:47:05,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025421 2024-11-24T13:47:05,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025421) roll requested 2024-11-24T13:47:05,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025429 2024-11-24T13:47:05,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025429 2024-11-24T13:47:05,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025429) roll requested 2024-11-24T13:47:05,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025437 2024-11-24T13:47:05,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025437 2024-11-24T13:47:05,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025437) roll requested 2024-11-24T13:47:05,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025445 2024-11-24T13:47:05,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025445 2024-11-24T13:47:05,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025445) roll requested 2024-11-24T13:47:05,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025455 2024-11-24T13:47:05,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025455 2024-11-24T13:47:05,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025455) roll requested 2024-11-24T13:47:05,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025467 2024-11-24T13:47:05,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025455 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025467 2024-11-24T13:47:05,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025467) roll requested 2024-11-24T13:47:05,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025473 2024-11-24T13:47:05,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025473 2024-11-24T13:47:05,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025473) roll requested 2024-11-24T13:47:05,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025478 2024-11-24T13:47:05,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025478 2024-11-24T13:47:05,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025478) roll requested 2024-11-24T13:47:05,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025484 2024-11-24T13:47:05,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025484 2024-11-24T13:47:05,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025484) roll requested 2024-11-24T13:47:05,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025488 2024-11-24T13:47:05,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025488 2024-11-24T13:47:05,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025488) roll requested 2024-11-24T13:47:05,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025494 2024-11-24T13:47:05,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025494 2024-11-24T13:47:05,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025494) roll requested 2024-11-24T13:47:05,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025501 2024-11-24T13:47:05,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025501 2024-11-24T13:47:05,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025501) roll requested 2024-11-24T13:47:05,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025509 2024-11-24T13:47:05,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025509 2024-11-24T13:47:05,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025509) roll requested 2024-11-24T13:47:05,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025518 2024-11-24T13:47:05,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025518 2024-11-24T13:47:05,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025518) roll requested 2024-11-24T13:47:05,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025526 2024-11-24T13:47:05,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025518 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025526 2024-11-24T13:47:05,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025526) roll requested 2024-11-24T13:47:05,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025536 2024-11-24T13:47:05,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025536 2024-11-24T13:47:05,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025536) roll requested 2024-11-24T13:47:05,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025543 2024-11-24T13:47:05,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025543 2024-11-24T13:47:05,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025543) roll requested 2024-11-24T13:47:05,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025556 2024-11-24T13:47:05,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025556 2024-11-24T13:47:05,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025556) roll requested 2024-11-24T13:47:05,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025566 2024-11-24T13:47:05,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025566 2024-11-24T13:47:05,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025566) roll requested 2024-11-24T13:47:05,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025580 2024-11-24T13:47:05,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025580 2024-11-24T13:47:05,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025580) roll requested 2024-11-24T13:47:05,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025589 2024-11-24T13:47:05,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025589 2024-11-24T13:47:05,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025589) roll requested 2024-11-24T13:47:05,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025594 2024-11-24T13:47:05,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025594 2024-11-24T13:47:05,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025594) roll requested 2024-11-24T13:47:05,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025599 2024-11-24T13:47:05,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025599 2024-11-24T13:47:05,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025599) roll requested 2024-11-24T13:47:05,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025605 2024-11-24T13:47:05,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025605 2024-11-24T13:47:05,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025605) roll requested 2024-11-24T13:47:05,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025611 2024-11-24T13:47:05,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025611 2024-11-24T13:47:05,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,619 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025611) roll requested 2024-11-24T13:47:05,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025619 2024-11-24T13:47:05,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025611 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025619 2024-11-24T13:47:05,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025619) roll requested 2024-11-24T13:47:05,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025627 2024-11-24T13:47:05,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025627 2024-11-24T13:47:05,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025627) roll requested 2024-11-24T13:47:05,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025634 2024-11-24T13:47:05,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025634 2024-11-24T13:47:05,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025634) roll requested 2024-11-24T13:47:05,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025642 2024-11-24T13:47:05,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025642 2024-11-24T13:47:05,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025642) roll requested 2024-11-24T13:47:05,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025650 2024-11-24T13:47:05,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025642 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025650 2024-11-24T13:47:05,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025650) roll requested 2024-11-24T13:47:05,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025659 2024-11-24T13:47:05,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025659 2024-11-24T13:47:05,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025659) roll requested 2024-11-24T13:47:05,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025667 2024-11-24T13:47:05,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025667 2024-11-24T13:47:05,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025667) roll requested 2024-11-24T13:47:05,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025672 2024-11-24T13:47:05,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025672 2024-11-24T13:47:05,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025672) roll requested 2024-11-24T13:47:05,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025678 2024-11-24T13:47:05,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025678 2024-11-24T13:47:05,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025678) roll requested 2024-11-24T13:47:05,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025683 2024-11-24T13:47:05,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025683 2024-11-24T13:47:05,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025683) roll requested 2024-11-24T13:47:05,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025689 2024-11-24T13:47:05,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025689 2024-11-24T13:47:05,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025689) roll requested 2024-11-24T13:47:05,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025697 2024-11-24T13:47:05,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025697 2024-11-24T13:47:05,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025697) roll requested 2024-11-24T13:47:05,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025703 2024-11-24T13:47:05,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025703 2024-11-24T13:47:05,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025703) roll requested 2024-11-24T13:47:05,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025711 2024-11-24T13:47:05,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025703 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025711 2024-11-24T13:47:05,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025711) roll requested 2024-11-24T13:47:05,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025720 2024-11-24T13:47:05,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025720 2024-11-24T13:47:05,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025720) roll requested 2024-11-24T13:47:05,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025729 2024-11-24T13:47:05,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025729 2024-11-24T13:47:05,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,738 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025729) roll requested 2024-11-24T13:47:05,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025738 2024-11-24T13:47:05,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025738 2024-11-24T13:47:05,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,746 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025738) roll requested 2024-11-24T13:47:05,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025746 2024-11-24T13:47:05,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025738 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025746 2024-11-24T13:47:05,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025746) roll requested 2024-11-24T13:47:05,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025753 2024-11-24T13:47:05,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025753 2024-11-24T13:47:05,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025753) roll requested 2024-11-24T13:47:05,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025761 2024-11-24T13:47:05,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025761 2024-11-24T13:47:05,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025761) roll requested 2024-11-24T13:47:05,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025772 2024-11-24T13:47:05,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025772 2024-11-24T13:47:05,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025772) roll requested 2024-11-24T13:47:05,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025785 2024-11-24T13:47:05,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025785 2024-11-24T13:47:05,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025785) roll requested 2024-11-24T13:47:05,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025791 2024-11-24T13:47:05,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025791 2024-11-24T13:47:05,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025791) roll requested 2024-11-24T13:47:05,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025797 2024-11-24T13:47:05,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025797 2024-11-24T13:47:05,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025797) roll requested 2024-11-24T13:47:05,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025801 2024-11-24T13:47:05,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025801 2024-11-24T13:47:05,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025801) roll requested 2024-11-24T13:47:05,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025807 2024-11-24T13:47:05,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025801 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025807 2024-11-24T13:47:05,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025807) roll requested 2024-11-24T13:47:05,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025813 2024-11-24T13:47:05,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025807 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025813 2024-11-24T13:47:05,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025813) roll requested 2024-11-24T13:47:05,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025820 2024-11-24T13:47:05,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025820 2024-11-24T13:47:05,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025820) roll requested 2024-11-24T13:47:05,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025829 2024-11-24T13:47:05,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025829 2024-11-24T13:47:05,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025829) roll requested 2024-11-24T13:47:05,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025837 2024-11-24T13:47:05,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025837 2024-11-24T13:47:05,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025837) roll requested 2024-11-24T13:47:05,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025845 2024-11-24T13:47:05,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025845 2024-11-24T13:47:05,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025845) roll requested 2024-11-24T13:47:05,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025854 2024-11-24T13:47:05,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025854 2024-11-24T13:47:05,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,863 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025854) roll requested 2024-11-24T13:47:05,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025863 2024-11-24T13:47:05,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025863 2024-11-24T13:47:05,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,870 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025863) roll requested 2024-11-24T13:47:05,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025870 2024-11-24T13:47:05,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025870 2024-11-24T13:47:05,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025870) roll requested 2024-11-24T13:47:05,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025876 2024-11-24T13:47:05,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025876 2024-11-24T13:47:05,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025876) roll requested 2024-11-24T13:47:05,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025884 2024-11-24T13:47:05,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025884 2024-11-24T13:47:05,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025884) roll requested 2024-11-24T13:47:05,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025890 2024-11-24T13:47:05,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025890 2024-11-24T13:47:05,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025890) roll requested 2024-11-24T13:47:05,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025896 2024-11-24T13:47:05,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025896 2024-11-24T13:47:05,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025896) roll requested 2024-11-24T13:47:05,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025902 2024-11-24T13:47:05,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025902 2024-11-24T13:47:05,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,909 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025902) roll requested 2024-11-24T13:47:05,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025909 2024-11-24T13:47:05,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025909 2024-11-24T13:47:05,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025909) roll requested 2024-11-24T13:47:05,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025917 2024-11-24T13:47:05,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025917 2024-11-24T13:47:05,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025917) roll requested 2024-11-24T13:47:05,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025926 2024-11-24T13:47:05,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025926 2024-11-24T13:47:05,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025926) roll requested 2024-11-24T13:47:05,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025933 2024-11-24T13:47:05,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025933 2024-11-24T13:47:05,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025933) roll requested 2024-11-24T13:47:05,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025942 2024-11-24T13:47:05,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025942 2024-11-24T13:47:05,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025942) roll requested 2024-11-24T13:47:05,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025954 2024-11-24T13:47:05,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025954 2024-11-24T13:47:05,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,967 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025954) roll requested 2024-11-24T13:47:05,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025967 2024-11-24T13:47:05,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025967 2024-11-24T13:47:05,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025967) roll requested 2024-11-24T13:47:05,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025973 2024-11-24T13:47:05,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025973 2024-11-24T13:47:05,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025973) roll requested 2024-11-24T13:47:05,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025987 2024-11-24T13:47:05,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025987 2024-11-24T13:47:05,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:05,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:05,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025987) roll requested 2024-11-24T13:47:05,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456025995 2024-11-24T13:47:05,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:05,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025995 2024-11-24T13:47:06,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456025995) roll requested 2024-11-24T13:47:06,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026003 2024-11-24T13:47:06,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456025995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026003 2024-11-24T13:47:06,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026003) roll requested 2024-11-24T13:47:06,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026010 2024-11-24T13:47:06,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026003 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026010 2024-11-24T13:47:06,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026010) roll requested 2024-11-24T13:47:06,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026023 2024-11-24T13:47:06,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026023 2024-11-24T13:47:06,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,037 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026023) roll requested 2024-11-24T13:47:06,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026037 2024-11-24T13:47:06,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026037 2024-11-24T13:47:06,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,046 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026037) roll requested 2024-11-24T13:47:06,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026047 2024-11-24T13:47:06,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026047 2024-11-24T13:47:06,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,053 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026047) roll requested 2024-11-24T13:47:06,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026053 2024-11-24T13:47:06,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026053 2024-11-24T13:47:06,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,067 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026053) roll requested 2024-11-24T13:47:06,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026067 2024-11-24T13:47:06,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026053 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026067 2024-11-24T13:47:06,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,079 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026067) roll requested 2024-11-24T13:47:06,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026080 2024-11-24T13:47:06,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026067 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026080 2024-11-24T13:47:06,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026080) roll requested 2024-11-24T13:47:06,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026092 2024-11-24T13:47:06,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026092 2024-11-24T13:47:06,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026092) roll requested 2024-11-24T13:47:06,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026103 2024-11-24T13:47:06,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026103 2024-11-24T13:47:06,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026103) roll requested 2024-11-24T13:47:06,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026118 2024-11-24T13:47:06,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026118 2024-11-24T13:47:06,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026118) roll requested 2024-11-24T13:47:06,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026131 2024-11-24T13:47:06,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026118 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026131 2024-11-24T13:47:06,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,138 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026131) roll requested 2024-11-24T13:47:06,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026139 2024-11-24T13:47:06,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026139 2024-11-24T13:47:06,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026139) roll requested 2024-11-24T13:47:06,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026151 2024-11-24T13:47:06,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026151 2024-11-24T13:47:06,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026151) roll requested 2024-11-24T13:47:06,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026159 2024-11-24T13:47:06,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026159 2024-11-24T13:47:06,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,166 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026159) roll requested 2024-11-24T13:47:06,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026166 2024-11-24T13:47:06,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026166 2024-11-24T13:47:06,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026166) roll requested 2024-11-24T13:47:06,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026175 2024-11-24T13:47:06,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026166 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026175 2024-11-24T13:47:06,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026175) roll requested 2024-11-24T13:47:06,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026183 2024-11-24T13:47:06,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026183 2024-11-24T13:47:06,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026183) roll requested 2024-11-24T13:47:06,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026190 2024-11-24T13:47:06,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026190 2024-11-24T13:47:06,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026190) roll requested 2024-11-24T13:47:06,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026197 2024-11-24T13:47:06,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026197 2024-11-24T13:47:06,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026197) roll requested 2024-11-24T13:47:06,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026205 2024-11-24T13:47:06,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026205 2024-11-24T13:47:06,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026205) roll requested 2024-11-24T13:47:06,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026212 2024-11-24T13:47:06,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026212 2024-11-24T13:47:06,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026212) roll requested 2024-11-24T13:47:06,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026218 2024-11-24T13:47:06,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026218 2024-11-24T13:47:06,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026218) roll requested 2024-11-24T13:47:06,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026225 2024-11-24T13:47:06,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026218 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026225 2024-11-24T13:47:06,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026225) roll requested 2024-11-24T13:47:06,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026234 2024-11-24T13:47:06,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026234 2024-11-24T13:47:06,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026234) roll requested 2024-11-24T13:47:06,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026240 2024-11-24T13:47:06,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026240 2024-11-24T13:47:06,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026240) roll requested 2024-11-24T13:47:06,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026246 2024-11-24T13:47:06,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026246 2024-11-24T13:47:06,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026246) roll requested 2024-11-24T13:47:06,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026251 2024-11-24T13:47:06,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026251 2024-11-24T13:47:06,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026251) roll requested 2024-11-24T13:47:06,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026256 2024-11-24T13:47:06,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026256 2024-11-24T13:47:06,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026256) roll requested 2024-11-24T13:47:06,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026263 2024-11-24T13:47:06,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026263 2024-11-24T13:47:06,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026263) roll requested 2024-11-24T13:47:06,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026271 2024-11-24T13:47:06,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026271 2024-11-24T13:47:06,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026271) roll requested 2024-11-24T13:47:06,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026279 2024-11-24T13:47:06,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026279 2024-11-24T13:47:06,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026279) roll requested 2024-11-24T13:47:06,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026286 2024-11-24T13:47:06,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026286 2024-11-24T13:47:06,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026286) roll requested 2024-11-24T13:47:06,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026294 2024-11-24T13:47:06,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026294 2024-11-24T13:47:06,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,302 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026294) roll requested 2024-11-24T13:47:06,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026302 2024-11-24T13:47:06,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026302 2024-11-24T13:47:06,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026302) roll requested 2024-11-24T13:47:06,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026310 2024-11-24T13:47:06,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026310 2024-11-24T13:47:06,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026310) roll requested 2024-11-24T13:47:06,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026318 2024-11-24T13:47:06,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026318 2024-11-24T13:47:06,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026318) roll requested 2024-11-24T13:47:06,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026325 2024-11-24T13:47:06,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026325 2024-11-24T13:47:06,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,330 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026325) roll requested 2024-11-24T13:47:06,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026330 2024-11-24T13:47:06,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026330 2024-11-24T13:47:06,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,335 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026330) roll requested 2024-11-24T13:47:06,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026335 2024-11-24T13:47:06,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026330 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026335 2024-11-24T13:47:06,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,342 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026335) roll requested 2024-11-24T13:47:06,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026342 2024-11-24T13:47:06,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026342 2024-11-24T13:47:06,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026342) roll requested 2024-11-24T13:47:06,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026349 2024-11-24T13:47:06,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026342 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026349 2024-11-24T13:47:06,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026349) roll requested 2024-11-24T13:47:06,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026356 2024-11-24T13:47:06,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026356 2024-11-24T13:47:06,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026356) roll requested 2024-11-24T13:47:06,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026363 2024-11-24T13:47:06,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026363 2024-11-24T13:47:06,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026363) roll requested 2024-11-24T13:47:06,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026371 2024-11-24T13:47:06,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026371 2024-11-24T13:47:06,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026371) roll requested 2024-11-24T13:47:06,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026379 2024-11-24T13:47:06,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026379 2024-11-24T13:47:06,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,388 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026379) roll requested 2024-11-24T13:47:06,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026388 2024-11-24T13:47:06,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026388 2024-11-24T13:47:06,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026388) roll requested 2024-11-24T13:47:06,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026426 2024-11-24T13:47:06,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026426 2024-11-24T13:47:06,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026426) roll requested 2024-11-24T13:47:06,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026448 2024-11-24T13:47:06,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026426 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026448 2024-11-24T13:47:06,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026448) roll requested 2024-11-24T13:47:06,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026467 2024-11-24T13:47:06,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026467 2024-11-24T13:47:06,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026467) roll requested 2024-11-24T13:47:06,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026487 2024-11-24T13:47:06,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026487 2024-11-24T13:47:06,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,498 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026487) roll requested 2024-11-24T13:47:06,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026498 2024-11-24T13:47:06,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026498 2024-11-24T13:47:06,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026498) roll requested 2024-11-24T13:47:06,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026505 2024-11-24T13:47:06,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026498 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026505 2024-11-24T13:47:06,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026505) roll requested 2024-11-24T13:47:06,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026513 2024-11-24T13:47:06,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026513 2024-11-24T13:47:06,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026513) roll requested 2024-11-24T13:47:06,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026522 2024-11-24T13:47:06,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026522 2024-11-24T13:47:06,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026522) roll requested 2024-11-24T13:47:06,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026528 2024-11-24T13:47:06,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026528 2024-11-24T13:47:06,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026528) roll requested 2024-11-24T13:47:06,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026541 2024-11-24T13:47:06,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026541 2024-11-24T13:47:06,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,551 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026541) roll requested 2024-11-24T13:47:06,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026552 2024-11-24T13:47:06,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026552 2024-11-24T13:47:06,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026552) roll requested 2024-11-24T13:47:06,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026557 2024-11-24T13:47:06,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026557 2024-11-24T13:47:06,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026557) roll requested 2024-11-24T13:47:06,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026565 2024-11-24T13:47:06,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026565 2024-11-24T13:47:06,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026565) roll requested 2024-11-24T13:47:06,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026571 2024-11-24T13:47:06,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026571 2024-11-24T13:47:06,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026571) roll requested 2024-11-24T13:47:06,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026577 2024-11-24T13:47:06,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026577 2024-11-24T13:47:06,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,582 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026577) roll requested 2024-11-24T13:47:06,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026582 2024-11-24T13:47:06,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026582 2024-11-24T13:47:06,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026582) roll requested 2024-11-24T13:47:06,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026588 2024-11-24T13:47:06,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026582 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026588 2024-11-24T13:47:06,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026588) roll requested 2024-11-24T13:47:06,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026593 2024-11-24T13:47:06,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026593 2024-11-24T13:47:06,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026593) roll requested 2024-11-24T13:47:06,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026599 2024-11-24T13:47:06,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026599 2024-11-24T13:47:06,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,606 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026599) roll requested 2024-11-24T13:47:06,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026606 2024-11-24T13:47:06,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026606 2024-11-24T13:47:06,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,613 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026606) roll requested 2024-11-24T13:47:06,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026614 2024-11-24T13:47:06,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026606 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026614 2024-11-24T13:47:06,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026614) roll requested 2024-11-24T13:47:06,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026621 2024-11-24T13:47:06,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026621 2024-11-24T13:47:06,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026621) roll requested 2024-11-24T13:47:06,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026630 2024-11-24T13:47:06,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026630 2024-11-24T13:47:06,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026630) roll requested 2024-11-24T13:47:06,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026636 2024-11-24T13:47:06,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026630 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026636 2024-11-24T13:47:06,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026636) roll requested 2024-11-24T13:47:06,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026644 2024-11-24T13:47:06,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026644 2024-11-24T13:47:06,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026644) roll requested 2024-11-24T13:47:06,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026654 2024-11-24T13:47:06,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026654 2024-11-24T13:47:06,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026654) roll requested 2024-11-24T13:47:06,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026664 2024-11-24T13:47:06,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026664 2024-11-24T13:47:06,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026664) roll requested 2024-11-24T13:47:06,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026673 2024-11-24T13:47:06,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026673 2024-11-24T13:47:06,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026673) roll requested 2024-11-24T13:47:06,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026681 2024-11-24T13:47:06,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026681 2024-11-24T13:47:06,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026681) roll requested 2024-11-24T13:47:06,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026689 2024-11-24T13:47:06,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026689 2024-11-24T13:47:06,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026689) roll requested 2024-11-24T13:47:06,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026694 2024-11-24T13:47:06,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026694 2024-11-24T13:47:06,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,699 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026694) roll requested 2024-11-24T13:47:06,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026699 2024-11-24T13:47:06,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026699 2024-11-24T13:47:06,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026699) roll requested 2024-11-24T13:47:06,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026706 2024-11-24T13:47:06,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026699 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026706 2024-11-24T13:47:06,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026706) roll requested 2024-11-24T13:47:06,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026712 2024-11-24T13:47:06,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026706 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026712 2024-11-24T13:47:06,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026712) roll requested 2024-11-24T13:47:06,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026720 2024-11-24T13:47:06,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026720 2024-11-24T13:47:06,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026720) roll requested 2024-11-24T13:47:06,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026727 2024-11-24T13:47:06,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026727 2024-11-24T13:47:06,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026727) roll requested 2024-11-24T13:47:06,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026733 2024-11-24T13:47:06,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026733 2024-11-24T13:47:06,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026733) roll requested 2024-11-24T13:47:06,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026739 2024-11-24T13:47:06,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026739 2024-11-24T13:47:06,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026739) roll requested 2024-11-24T13:47:06,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026750 2024-11-24T13:47:06,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026750 2024-11-24T13:47:06,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026750) roll requested 2024-11-24T13:47:06,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026756 2024-11-24T13:47:06,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026756 2024-11-24T13:47:06,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,762 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026756) roll requested 2024-11-24T13:47:06,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026763 2024-11-24T13:47:06,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026763 2024-11-24T13:47:06,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026763) roll requested 2024-11-24T13:47:06,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026775 2024-11-24T13:47:06,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026775 2024-11-24T13:47:06,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026775) roll requested 2024-11-24T13:47:06,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026783 2024-11-24T13:47:06,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026783 2024-11-24T13:47:06,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026783) roll requested 2024-11-24T13:47:06,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026788 2024-11-24T13:47:06,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026788 2024-11-24T13:47:06,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,794 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026788) roll requested 2024-11-24T13:47:06,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026794 2024-11-24T13:47:06,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026794 2024-11-24T13:47:06,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026794) roll requested 2024-11-24T13:47:06,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026803 2024-11-24T13:47:06,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026803 2024-11-24T13:47:06,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026803) roll requested 2024-11-24T13:47:06,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026808 2024-11-24T13:47:06,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026808 2024-11-24T13:47:06,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026808) roll requested 2024-11-24T13:47:06,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026813 2024-11-24T13:47:06,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026813 2024-11-24T13:47:06,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026813) roll requested 2024-11-24T13:47:06,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026818 2024-11-24T13:47:06,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026818 2024-11-24T13:47:06,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,824 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026818) roll requested 2024-11-24T13:47:06,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026824 2024-11-24T13:47:06,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026824 2024-11-24T13:47:06,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026824) roll requested 2024-11-24T13:47:06,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026831 2024-11-24T13:47:06,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026824 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026831 2024-11-24T13:47:06,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026831) roll requested 2024-11-24T13:47:06,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026837 2024-11-24T13:47:06,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026837 2024-11-24T13:47:06,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026837) roll requested 2024-11-24T13:47:06,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026843 2024-11-24T13:47:06,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026843 2024-11-24T13:47:06,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026843) roll requested 2024-11-24T13:47:06,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026848 2024-11-24T13:47:06,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026843 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026848 2024-11-24T13:47:06,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026848) roll requested 2024-11-24T13:47:06,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026855 2024-11-24T13:47:06,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026855 2024-11-24T13:47:06,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026855) roll requested 2024-11-24T13:47:06,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026860 2024-11-24T13:47:06,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026860 2024-11-24T13:47:06,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026860) roll requested 2024-11-24T13:47:06,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026866 2024-11-24T13:47:06,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026866 2024-11-24T13:47:06,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026866) roll requested 2024-11-24T13:47:06,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026871 2024-11-24T13:47:06,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026866 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026871 2024-11-24T13:47:06,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026871) roll requested 2024-11-24T13:47:06,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026876 2024-11-24T13:47:06,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026876 2024-11-24T13:47:06,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026876) roll requested 2024-11-24T13:47:06,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026881 2024-11-24T13:47:06,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026881 2024-11-24T13:47:06,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,886 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026881) roll requested 2024-11-24T13:47:06,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026886 2024-11-24T13:47:06,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026886 2024-11-24T13:47:06,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,891 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026886) roll requested 2024-11-24T13:47:06,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026891 2024-11-24T13:47:06,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026886 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026891 2024-11-24T13:47:06,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,897 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026891) roll requested 2024-11-24T13:47:06,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026897 2024-11-24T13:47:06,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026897 2024-11-24T13:47:06,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026897) roll requested 2024-11-24T13:47:06,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026905 2024-11-24T13:47:06,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026897 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026905 2024-11-24T13:47:06,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,915 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026905) roll requested 2024-11-24T13:47:06,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026915 2024-11-24T13:47:06,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026915 2024-11-24T13:47:06,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,922 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026915) roll requested 2024-11-24T13:47:06,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026922 2024-11-24T13:47:06,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026922 2024-11-24T13:47:06,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,930 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026922) roll requested 2024-11-24T13:47:06,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026931 2024-11-24T13:47:06,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026931 2024-11-24T13:47:06,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026931) roll requested 2024-11-24T13:47:06,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026936 2024-11-24T13:47:06,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026936 2024-11-24T13:47:06,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026936) roll requested 2024-11-24T13:47:06,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026942 2024-11-24T13:47:06,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026942 2024-11-24T13:47:06,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026942) roll requested 2024-11-24T13:47:06,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026947 2024-11-24T13:47:06,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026947 2024-11-24T13:47:06,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026947) roll requested 2024-11-24T13:47:06,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026952 2024-11-24T13:47:06,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026952 2024-11-24T13:47:06,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026952) roll requested 2024-11-24T13:47:06,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026958 2024-11-24T13:47:06,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026958 2024-11-24T13:47:06,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026958) roll requested 2024-11-24T13:47:06,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026964 2024-11-24T13:47:06,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026964 2024-11-24T13:47:06,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026964) roll requested 2024-11-24T13:47:06,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026972 2024-11-24T13:47:06,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026972 2024-11-24T13:47:06,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026972) roll requested 2024-11-24T13:47:06,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026980 2024-11-24T13:47:06,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026980 2024-11-24T13:47:06,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026980) roll requested 2024-11-24T13:47:06,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026989 2024-11-24T13:47:06,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:06,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026989 2024-11-24T13:47:06,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:06,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:06,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026989) roll requested 2024-11-24T13:47:06,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456026998 2024-11-24T13:47:07,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026998 2024-11-24T13:47:07,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456026998) roll requested 2024-11-24T13:47:07,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027008 2024-11-24T13:47:07,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456026998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027008 2024-11-24T13:47:07,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027008) roll requested 2024-11-24T13:47:07,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027016 2024-11-24T13:47:07,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027016 2024-11-24T13:47:07,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027016) roll requested 2024-11-24T13:47:07,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027027 2024-11-24T13:47:07,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027027 2024-11-24T13:47:07,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027027) roll requested 2024-11-24T13:47:07,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027035 2024-11-24T13:47:07,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027035 2024-11-24T13:47:07,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027035) roll requested 2024-11-24T13:47:07,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027043 2024-11-24T13:47:07,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027043 2024-11-24T13:47:07,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027043) roll requested 2024-11-24T13:47:07,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027051 2024-11-24T13:47:07,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027051 2024-11-24T13:47:07,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027051) roll requested 2024-11-24T13:47:07,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027061 2024-11-24T13:47:07,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027061 2024-11-24T13:47:07,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027061) roll requested 2024-11-24T13:47:07,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027073 2024-11-24T13:47:07,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027073 2024-11-24T13:47:07,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027073) roll requested 2024-11-24T13:47:07,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027089 2024-11-24T13:47:07,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027089 2024-11-24T13:47:07,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027089) roll requested 2024-11-24T13:47:07,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027104 2024-11-24T13:47:07,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027104 2024-11-24T13:47:07,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,110 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027104) roll requested 2024-11-24T13:47:07,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027110 2024-11-24T13:47:07,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027104 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027110 2024-11-24T13:47:07,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,117 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027110) roll requested 2024-11-24T13:47:07,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027117 2024-11-24T13:47:07,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027110 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027117 2024-11-24T13:47:07,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027117) roll requested 2024-11-24T13:47:07,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027123 2024-11-24T13:47:07,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027123 2024-11-24T13:47:07,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027123) roll requested 2024-11-24T13:47:07,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027129 2024-11-24T13:47:07,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027129 2024-11-24T13:47:07,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027129) roll requested 2024-11-24T13:47:07,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027134 2024-11-24T13:47:07,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027134 2024-11-24T13:47:07,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,139 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027134) roll requested 2024-11-24T13:47:07,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027139 2024-11-24T13:47:07,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027139 2024-11-24T13:47:07,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027139) roll requested 2024-11-24T13:47:07,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027145 2024-11-24T13:47:07,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027145 2024-11-24T13:47:07,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027145) roll requested 2024-11-24T13:47:07,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027153 2024-11-24T13:47:07,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027153 2024-11-24T13:47:07,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,163 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027153) roll requested 2024-11-24T13:47:07,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027163 2024-11-24T13:47:07,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027163 2024-11-24T13:47:07,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027163) roll requested 2024-11-24T13:47:07,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027171 2024-11-24T13:47:07,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027163 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027171 2024-11-24T13:47:07,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027171) roll requested 2024-11-24T13:47:07,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027188 2024-11-24T13:47:07,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027188 2024-11-24T13:47:07,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027188) roll requested 2024-11-24T13:47:07,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027199 2024-11-24T13:47:07,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027199 2024-11-24T13:47:07,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027199) roll requested 2024-11-24T13:47:07,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027206 2024-11-24T13:47:07,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027199 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027206 2024-11-24T13:47:07,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027206) roll requested 2024-11-24T13:47:07,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027213 2024-11-24T13:47:07,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027213 2024-11-24T13:47:07,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027213) roll requested 2024-11-24T13:47:07,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027222 2024-11-24T13:47:07,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027222 2024-11-24T13:47:07,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027222) roll requested 2024-11-24T13:47:07,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027231 2024-11-24T13:47:07,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027231 2024-11-24T13:47:07,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027231) roll requested 2024-11-24T13:47:07,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027236 2024-11-24T13:47:07,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027236 2024-11-24T13:47:07,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027236) roll requested 2024-11-24T13:47:07,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027242 2024-11-24T13:47:07,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027236 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027242 2024-11-24T13:47:07,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027242) roll requested 2024-11-24T13:47:07,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027249 2024-11-24T13:47:07,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027249 2024-11-24T13:47:07,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027249) roll requested 2024-11-24T13:47:07,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027256 2024-11-24T13:47:07,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027256 2024-11-24T13:47:07,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027256) roll requested 2024-11-24T13:47:07,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027262 2024-11-24T13:47:07,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027262 2024-11-24T13:47:07,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027262) roll requested 2024-11-24T13:47:07,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027270 2024-11-24T13:47:07,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027270 2024-11-24T13:47:07,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027270) roll requested 2024-11-24T13:47:07,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027281 2024-11-24T13:47:07,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027281 2024-11-24T13:47:07,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,290 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027281) roll requested 2024-11-24T13:47:07,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027291 2024-11-24T13:47:07,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027291 2024-11-24T13:47:07,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,302 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027291) roll requested 2024-11-24T13:47:07,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027302 2024-11-24T13:47:07,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027291 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027302 2024-11-24T13:47:07,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027302) roll requested 2024-11-24T13:47:07,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027311 2024-11-24T13:47:07,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027311 2024-11-24T13:47:07,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027311) roll requested 2024-11-24T13:47:07,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027318 2024-11-24T13:47:07,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027311 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027318 2024-11-24T13:47:07,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027318) roll requested 2024-11-24T13:47:07,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027327 2024-11-24T13:47:07,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027327 2024-11-24T13:47:07,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,335 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027327) roll requested 2024-11-24T13:47:07,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027335 2024-11-24T13:47:07,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027335 2024-11-24T13:47:07,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,342 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027335) roll requested 2024-11-24T13:47:07,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027343 2024-11-24T13:47:07,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027343 2024-11-24T13:47:07,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027343) roll requested 2024-11-24T13:47:07,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027354 2024-11-24T13:47:07,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027354 2024-11-24T13:47:07,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,367 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027354) roll requested 2024-11-24T13:47:07,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027368 2024-11-24T13:47:07,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027368 2024-11-24T13:47:07,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027368) roll requested 2024-11-24T13:47:07,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027375 2024-11-24T13:47:07,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027375 2024-11-24T13:47:07,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,381 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027375) roll requested 2024-11-24T13:47:07,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027381 2024-11-24T13:47:07,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027381 2024-11-24T13:47:07,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027381) roll requested 2024-11-24T13:47:07,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027395 2024-11-24T13:47:07,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027381 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027395 2024-11-24T13:47:07,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,407 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027395) roll requested 2024-11-24T13:47:07,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027407 2024-11-24T13:47:07,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027407 2024-11-24T13:47:07,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027407) roll requested 2024-11-24T13:47:07,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027413 2024-11-24T13:47:07,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027413 2024-11-24T13:47:07,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,419 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027413) roll requested 2024-11-24T13:47:07,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027419 2024-11-24T13:47:07,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027419 2024-11-24T13:47:07,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,425 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027419) roll requested 2024-11-24T13:47:07,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027425 2024-11-24T13:47:07,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027425 2024-11-24T13:47:07,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027425) roll requested 2024-11-24T13:47:07,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027435 2024-11-24T13:47:07,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027435 2024-11-24T13:47:07,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027435) roll requested 2024-11-24T13:47:07,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027440 2024-11-24T13:47:07,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027440 2024-11-24T13:47:07,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027440) roll requested 2024-11-24T13:47:07,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027449 2024-11-24T13:47:07,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027449 2024-11-24T13:47:07,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,454 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027449) roll requested 2024-11-24T13:47:07,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027454 2024-11-24T13:47:07,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027454 2024-11-24T13:47:07,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027454) roll requested 2024-11-24T13:47:07,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027466 2024-11-24T13:47:07,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027466 2024-11-24T13:47:07,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027466) roll requested 2024-11-24T13:47:07,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027473 2024-11-24T13:47:07,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027473 2024-11-24T13:47:07,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027473) roll requested 2024-11-24T13:47:07,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027480 2024-11-24T13:47:07,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027480 2024-11-24T13:47:07,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027480) roll requested 2024-11-24T13:47:07,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027486 2024-11-24T13:47:07,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027486 2024-11-24T13:47:07,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027486) roll requested 2024-11-24T13:47:07,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027493 2024-11-24T13:47:07,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027493 2024-11-24T13:47:07,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,499 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027493) roll requested 2024-11-24T13:47:07,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027499 2024-11-24T13:47:07,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027499 2024-11-24T13:47:07,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027499) roll requested 2024-11-24T13:47:07,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027507 2024-11-24T13:47:07,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027499 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027507 2024-11-24T13:47:07,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027507) roll requested 2024-11-24T13:47:07,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027512 2024-11-24T13:47:07,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027512 2024-11-24T13:47:07,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027512) roll requested 2024-11-24T13:47:07,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027516 2024-11-24T13:47:07,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027516 2024-11-24T13:47:07,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027516) roll requested 2024-11-24T13:47:07,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027522 2024-11-24T13:47:07,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027516 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027522 2024-11-24T13:47:07,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027522) roll requested 2024-11-24T13:47:07,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027528 2024-11-24T13:47:07,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027528 2024-11-24T13:47:07,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027528) roll requested 2024-11-24T13:47:07,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027535 2024-11-24T13:47:07,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027535 2024-11-24T13:47:07,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027535) roll requested 2024-11-24T13:47:07,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027542 2024-11-24T13:47:07,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027542 2024-11-24T13:47:07,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027542) roll requested 2024-11-24T13:47:07,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027547 2024-11-24T13:47:07,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027547 2024-11-24T13:47:07,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027547) roll requested 2024-11-24T13:47:07,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027553 2024-11-24T13:47:07,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027553 2024-11-24T13:47:07,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027553) roll requested 2024-11-24T13:47:07,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027560 2024-11-24T13:47:07,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027560 2024-11-24T13:47:07,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027560) roll requested 2024-11-24T13:47:07,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027575 2024-11-24T13:47:07,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027575 2024-11-24T13:47:07,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027575) roll requested 2024-11-24T13:47:07,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027580 2024-11-24T13:47:07,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027575 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027580 2024-11-24T13:47:07,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,590 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027580) roll requested 2024-11-24T13:47:07,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027591 2024-11-24T13:47:07,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027591 2024-11-24T13:47:07,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027591) roll requested 2024-11-24T13:47:07,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027601 2024-11-24T13:47:07,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027591 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027601 2024-11-24T13:47:07,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,606 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027601) roll requested 2024-11-24T13:47:07,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027607 2024-11-24T13:47:07,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027607 2024-11-24T13:47:07,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027607) roll requested 2024-11-24T13:47:07,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027613 2024-11-24T13:47:07,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027607 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027613 2024-11-24T13:47:07,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027613) roll requested 2024-11-24T13:47:07,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027619 2024-11-24T13:47:07,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027613 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027619 2024-11-24T13:47:07,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027619) roll requested 2024-11-24T13:47:07,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027625 2024-11-24T13:47:07,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027625 2024-11-24T13:47:07,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,630 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027625) roll requested 2024-11-24T13:47:07,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027630 2024-11-24T13:47:07,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027630 2024-11-24T13:47:07,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,635 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027630) roll requested 2024-11-24T13:47:07,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027635 2024-11-24T13:47:07,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027630 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027635 2024-11-24T13:47:07,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027635) roll requested 2024-11-24T13:47:07,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027642 2024-11-24T13:47:07,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027642 2024-11-24T13:47:07,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027642) roll requested 2024-11-24T13:47:07,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027648 2024-11-24T13:47:07,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027642 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027648 2024-11-24T13:47:07,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027648) roll requested 2024-11-24T13:47:07,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027657 2024-11-24T13:47:07,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027657 2024-11-24T13:47:07,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027657) roll requested 2024-11-24T13:47:07,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027663 2024-11-24T13:47:07,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027663 2024-11-24T13:47:07,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027663) roll requested 2024-11-24T13:47:07,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027668 2024-11-24T13:47:07,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027663 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027668 2024-11-24T13:47:07,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027668) roll requested 2024-11-24T13:47:07,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027675 2024-11-24T13:47:07,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027675 2024-11-24T13:47:07,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027675) roll requested 2024-11-24T13:47:07,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027680 2024-11-24T13:47:07,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027675 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027680 2024-11-24T13:47:07,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027680) roll requested 2024-11-24T13:47:07,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027685 2024-11-24T13:47:07,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027685 2024-11-24T13:47:07,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027685) roll requested 2024-11-24T13:47:07,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027694 2024-11-24T13:47:07,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027694 2024-11-24T13:47:07,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027694) roll requested 2024-11-24T13:47:07,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027700 2024-11-24T13:47:07,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027700 2024-11-24T13:47:07,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027700) roll requested 2024-11-24T13:47:07,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027709 2024-11-24T13:47:07,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027709 2024-11-24T13:47:07,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027709) roll requested 2024-11-24T13:47:07,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027714 2024-11-24T13:47:07,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027714 2024-11-24T13:47:07,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027714) roll requested 2024-11-24T13:47:07,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027720 2024-11-24T13:47:07,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027720 2024-11-24T13:47:07,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027720) roll requested 2024-11-24T13:47:07,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027726 2024-11-24T13:47:07,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027726 2024-11-24T13:47:07,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027726) roll requested 2024-11-24T13:47:07,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027732 2024-11-24T13:47:07,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027732 2024-11-24T13:47:07,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,738 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027732) roll requested 2024-11-24T13:47:07,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027738 2024-11-24T13:47:07,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027738 2024-11-24T13:47:07,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027738) roll requested 2024-11-24T13:47:07,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027744 2024-11-24T13:47:07,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027738 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027744 2024-11-24T13:47:07,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027744) roll requested 2024-11-24T13:47:07,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027750 2024-11-24T13:47:07,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027750 2024-11-24T13:47:07,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,754 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027750) roll requested 2024-11-24T13:47:07,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027754 2024-11-24T13:47:07,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027754 2024-11-24T13:47:07,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027754) roll requested 2024-11-24T13:47:07,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027759 2024-11-24T13:47:07,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027754 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027759 2024-11-24T13:47:07,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027759) roll requested 2024-11-24T13:47:07,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027765 2024-11-24T13:47:07,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027765 2024-11-24T13:47:07,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027765) roll requested 2024-11-24T13:47:07,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027770 2024-11-24T13:47:07,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027770 2024-11-24T13:47:07,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027770) roll requested 2024-11-24T13:47:07,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027775 2024-11-24T13:47:07,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027775 2024-11-24T13:47:07,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027775) roll requested 2024-11-24T13:47:07,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027781 2024-11-24T13:47:07,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027781 2024-11-24T13:47:07,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,787 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027781) roll requested 2024-11-24T13:47:07,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027788 2024-11-24T13:47:07,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027788 2024-11-24T13:47:07,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,793 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027788) roll requested 2024-11-24T13:47:07,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027793 2024-11-24T13:47:07,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027793 2024-11-24T13:47:07,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027793) roll requested 2024-11-24T13:47:07,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027799 2024-11-24T13:47:07,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027799 2024-11-24T13:47:07,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027799) roll requested 2024-11-24T13:47:07,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027805 2024-11-24T13:47:07,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027805 2024-11-24T13:47:07,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027805) roll requested 2024-11-24T13:47:07,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027814 2024-11-24T13:47:07,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027814 2024-11-24T13:47:07,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,824 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027814) roll requested 2024-11-24T13:47:07,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027824 2024-11-24T13:47:07,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027824 2024-11-24T13:47:07,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027824) roll requested 2024-11-24T13:47:07,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027834 2024-11-24T13:47:07,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027824 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027834 2024-11-24T13:47:07,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027834) roll requested 2024-11-24T13:47:07,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027841 2024-11-24T13:47:07,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027841 2024-11-24T13:47:07,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027841) roll requested 2024-11-24T13:47:07,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027851 2024-11-24T13:47:07,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027851 2024-11-24T13:47:07,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027851) roll requested 2024-11-24T13:47:07,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027857 2024-11-24T13:47:07,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027857 2024-11-24T13:47:07,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027857) roll requested 2024-11-24T13:47:07,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027862 2024-11-24T13:47:07,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027862 2024-11-24T13:47:07,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027862) roll requested 2024-11-24T13:47:07,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027868 2024-11-24T13:47:07,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027868 2024-11-24T13:47:07,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,874 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027868) roll requested 2024-11-24T13:47:07,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027874 2024-11-24T13:47:07,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027874 2024-11-24T13:47:07,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027874) roll requested 2024-11-24T13:47:07,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027881 2024-11-24T13:47:07,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027881 2024-11-24T13:47:07,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027881) roll requested 2024-11-24T13:47:07,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027888 2024-11-24T13:47:07,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027888 2024-11-24T13:47:07,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027888) roll requested 2024-11-24T13:47:07,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027898 2024-11-24T13:47:07,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027898 2024-11-24T13:47:07,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027898) roll requested 2024-11-24T13:47:07,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027909 2024-11-24T13:47:07,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027909 2024-11-24T13:47:07,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027909) roll requested 2024-11-24T13:47:07,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027914 2024-11-24T13:47:07,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027914 2024-11-24T13:47:07,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027914) roll requested 2024-11-24T13:47:07,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027919 2024-11-24T13:47:07,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027919 2024-11-24T13:47:07,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027919) roll requested 2024-11-24T13:47:07,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027925 2024-11-24T13:47:07,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027925 2024-11-24T13:47:07,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027925) roll requested 2024-11-24T13:47:07,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027932 2024-11-24T13:47:07,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027932 2024-11-24T13:47:07,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027932) roll requested 2024-11-24T13:47:07,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027940 2024-11-24T13:47:07,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027940 2024-11-24T13:47:07,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027940) roll requested 2024-11-24T13:47:07,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027949 2024-11-24T13:47:07,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027949 2024-11-24T13:47:07,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027949) roll requested 2024-11-24T13:47:07,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027957 2024-11-24T13:47:07,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027957 2024-11-24T13:47:07,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027957) roll requested 2024-11-24T13:47:07,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027965 2024-11-24T13:47:07,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027957 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027965 2024-11-24T13:47:07,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027965) roll requested 2024-11-24T13:47:07,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027973 2024-11-24T13:47:07,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027973 2024-11-24T13:47:07,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027973) roll requested 2024-11-24T13:47:07,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027983 2024-11-24T13:47:07,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027983 2024-11-24T13:47:07,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027983) roll requested 2024-11-24T13:47:07,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027992 2024-11-24T13:47:07,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:07,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027992 2024-11-24T13:47:07,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:07,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:07,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027992) roll requested 2024-11-24T13:47:07,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456027997 2024-11-24T13:47:08,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027997 2024-11-24T13:47:08,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,002 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456027997) roll requested 2024-11-24T13:47:08,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028003 2024-11-24T13:47:08,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456027997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028003 2024-11-24T13:47:08,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028003) roll requested 2024-11-24T13:47:08,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028009 2024-11-24T13:47:08,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028003 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028009 2024-11-24T13:47:08,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028009) roll requested 2024-11-24T13:47:08,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028017 2024-11-24T13:47:08,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028017 2024-11-24T13:47:08,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028017) roll requested 2024-11-24T13:47:08,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028025 2024-11-24T13:47:08,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028025 2024-11-24T13:47:08,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028025) roll requested 2024-11-24T13:47:08,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028034 2024-11-24T13:47:08,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028034 2024-11-24T13:47:08,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,042 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028034) roll requested 2024-11-24T13:47:08,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028043 2024-11-24T13:47:08,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028043 2024-11-24T13:47:08,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,050 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028043) roll requested 2024-11-24T13:47:08,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028050 2024-11-24T13:47:08,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028050 2024-11-24T13:47:08,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028050) roll requested 2024-11-24T13:47:08,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028057 2024-11-24T13:47:08,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028050 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028057 2024-11-24T13:47:08,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028057) roll requested 2024-11-24T13:47:08,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028065 2024-11-24T13:47:08,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028065 2024-11-24T13:47:08,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028065) roll requested 2024-11-24T13:47:08,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028074 2024-11-24T13:47:08,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028074 2024-11-24T13:47:08,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028074) roll requested 2024-11-24T13:47:08,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028086 2024-11-24T13:47:08,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028074 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028086 2024-11-24T13:47:08,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,095 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028086) roll requested 2024-11-24T13:47:08,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028095 2024-11-24T13:47:08,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028095 2024-11-24T13:47:08,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028095) roll requested 2024-11-24T13:47:08,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028101 2024-11-24T13:47:08,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028095 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028101 2024-11-24T13:47:08,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028101) roll requested 2024-11-24T13:47:08,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028107 2024-11-24T13:47:08,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028107 2024-11-24T13:47:08,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028107) roll requested 2024-11-24T13:47:08,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028115 2024-11-24T13:47:08,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028115 2024-11-24T13:47:08,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028115) roll requested 2024-11-24T13:47:08,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028123 2024-11-24T13:47:08,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028115 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028123 2024-11-24T13:47:08,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028123) roll requested 2024-11-24T13:47:08,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028129 2024-11-24T13:47:08,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028129 2024-11-24T13:47:08,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028129) roll requested 2024-11-24T13:47:08,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028136 2024-11-24T13:47:08,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028136 2024-11-24T13:47:08,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028136) roll requested 2024-11-24T13:47:08,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028144 2024-11-24T13:47:08,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028144 2024-11-24T13:47:08,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028144) roll requested 2024-11-24T13:47:08,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028153 2024-11-24T13:47:08,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028144 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028153 2024-11-24T13:47:08,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028153) roll requested 2024-11-24T13:47:08,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028161 2024-11-24T13:47:08,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028161 2024-11-24T13:47:08,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028161) roll requested 2024-11-24T13:47:08,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028168 2024-11-24T13:47:08,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028168 2024-11-24T13:47:08,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028168) roll requested 2024-11-24T13:47:08,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028176 2024-11-24T13:47:08,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028176 2024-11-24T13:47:08,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028176) roll requested 2024-11-24T13:47:08,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028185 2024-11-24T13:47:08,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028185 2024-11-24T13:47:08,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028185) roll requested 2024-11-24T13:47:08,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028194 2024-11-24T13:47:08,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028194 2024-11-24T13:47:08,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028194) roll requested 2024-11-24T13:47:08,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028203 2024-11-24T13:47:08,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028194 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028203 2024-11-24T13:47:08,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028203) roll requested 2024-11-24T13:47:08,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028214 2024-11-24T13:47:08,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028203 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028214 2024-11-24T13:47:08,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028214) roll requested 2024-11-24T13:47:08,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028222 2024-11-24T13:47:08,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028222 2024-11-24T13:47:08,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028222) roll requested 2024-11-24T13:47:08,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028228 2024-11-24T13:47:08,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028228 2024-11-24T13:47:08,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,234 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028228) roll requested 2024-11-24T13:47:08,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028234 2024-11-24T13:47:08,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028234 2024-11-24T13:47:08,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028234) roll requested 2024-11-24T13:47:08,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028241 2024-11-24T13:47:08,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028241 2024-11-24T13:47:08,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028241) roll requested 2024-11-24T13:47:08,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028249 2024-11-24T13:47:08,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028249 2024-11-24T13:47:08,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028249) roll requested 2024-11-24T13:47:08,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028257 2024-11-24T13:47:08,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028257 2024-11-24T13:47:08,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,265 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028257) roll requested 2024-11-24T13:47:08,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028266 2024-11-24T13:47:08,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028266 2024-11-24T13:47:08,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028266) roll requested 2024-11-24T13:47:08,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028274 2024-11-24T13:47:08,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028274 2024-11-24T13:47:08,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028274) roll requested 2024-11-24T13:47:08,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028281 2024-11-24T13:47:08,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028281 2024-11-24T13:47:08,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028281) roll requested 2024-11-24T13:47:08,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028288 2024-11-24T13:47:08,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028288 2024-11-24T13:47:08,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028288) roll requested 2024-11-24T13:47:08,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028296 2024-11-24T13:47:08,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028296 2024-11-24T13:47:08,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028296) roll requested 2024-11-24T13:47:08,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028305 2024-11-24T13:47:08,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028296 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028305 2024-11-24T13:47:08,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028305) roll requested 2024-11-24T13:47:08,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028314 2024-11-24T13:47:08,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028314 2024-11-24T13:47:08,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028314) roll requested 2024-11-24T13:47:08,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028324 2024-11-24T13:47:08,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028324 2024-11-24T13:47:08,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028324) roll requested 2024-11-24T13:47:08,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028330 2024-11-24T13:47:08,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028330 2024-11-24T13:47:08,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,335 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028330) roll requested 2024-11-24T13:47:08,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028335 2024-11-24T13:47:08,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028330 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028335 2024-11-24T13:47:08,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028335) roll requested 2024-11-24T13:47:08,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028341 2024-11-24T13:47:08,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028341 2024-11-24T13:47:08,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028341) roll requested 2024-11-24T13:47:08,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028348 2024-11-24T13:47:08,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028348 2024-11-24T13:47:08,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,357 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028348) roll requested 2024-11-24T13:47:08,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028357 2024-11-24T13:47:08,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028357 2024-11-24T13:47:08,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028357) roll requested 2024-11-24T13:47:08,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028366 2024-11-24T13:47:08,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028366 2024-11-24T13:47:08,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028366) roll requested 2024-11-24T13:47:08,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028376 2024-11-24T13:47:08,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028376 2024-11-24T13:47:08,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,381 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028376) roll requested 2024-11-24T13:47:08,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028381 2024-11-24T13:47:08,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028381 2024-11-24T13:47:08,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028381) roll requested 2024-11-24T13:47:08,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028389 2024-11-24T13:47:08,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028381 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028389 2024-11-24T13:47:08,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028389) roll requested 2024-11-24T13:47:08,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028399 2024-11-24T13:47:08,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028399 2024-11-24T13:47:08,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028399) roll requested 2024-11-24T13:47:08,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028407 2024-11-24T13:47:08,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028407 2024-11-24T13:47:08,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028407) roll requested 2024-11-24T13:47:08,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028416 2024-11-24T13:47:08,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028416 2024-11-24T13:47:08,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028416) roll requested 2024-11-24T13:47:08,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028422 2024-11-24T13:47:08,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028422 2024-11-24T13:47:08,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028422) roll requested 2024-11-24T13:47:08,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028427 2024-11-24T13:47:08,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028427 2024-11-24T13:47:08,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028427) roll requested 2024-11-24T13:47:08,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028433 2024-11-24T13:47:08,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028433 2024-11-24T13:47:08,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028433) roll requested 2024-11-24T13:47:08,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028438 2024-11-24T13:47:08,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028438 2024-11-24T13:47:08,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,443 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028438) roll requested 2024-11-24T13:47:08,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028444 2024-11-24T13:47:08,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028444 2024-11-24T13:47:08,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028444) roll requested 2024-11-24T13:47:08,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028449 2024-11-24T13:47:08,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028449 2024-11-24T13:47:08,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028449) roll requested 2024-11-24T13:47:08,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028457 2024-11-24T13:47:08,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028457 2024-11-24T13:47:08,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028457) roll requested 2024-11-24T13:47:08,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028465 2024-11-24T13:47:08,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028465 2024-11-24T13:47:08,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028465) roll requested 2024-11-24T13:47:08,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028477 2024-11-24T13:47:08,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028477 2024-11-24T13:47:08,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028477) roll requested 2024-11-24T13:47:08,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028482 2024-11-24T13:47:08,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028482 2024-11-24T13:47:08,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028482) roll requested 2024-11-24T13:47:08,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028487 2024-11-24T13:47:08,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028482 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028487 2024-11-24T13:47:08,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028487) roll requested 2024-11-24T13:47:08,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028493 2024-11-24T13:47:08,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028493 2024-11-24T13:47:08,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,499 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028493) roll requested 2024-11-24T13:47:08,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028499 2024-11-24T13:47:08,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028499 2024-11-24T13:47:08,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028499) roll requested 2024-11-24T13:47:08,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028507 2024-11-24T13:47:08,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028499 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028507 2024-11-24T13:47:08,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028507) roll requested 2024-11-24T13:47:08,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028514 2024-11-24T13:47:08,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028514 2024-11-24T13:47:08,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028514) roll requested 2024-11-24T13:47:08,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028520 2024-11-24T13:47:08,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028520 2024-11-24T13:47:08,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028520) roll requested 2024-11-24T13:47:08,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028526 2024-11-24T13:47:08,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028526 2024-11-24T13:47:08,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028526) roll requested 2024-11-24T13:47:08,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028533 2024-11-24T13:47:08,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028533 2024-11-24T13:47:08,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028533) roll requested 2024-11-24T13:47:08,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028541 2024-11-24T13:47:08,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028541 2024-11-24T13:47:08,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028541) roll requested 2024-11-24T13:47:08,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028549 2024-11-24T13:47:08,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028549 2024-11-24T13:47:08,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028549) roll requested 2024-11-24T13:47:08,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028559 2024-11-24T13:47:08,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028559 2024-11-24T13:47:08,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028559) roll requested 2024-11-24T13:47:08,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028568 2024-11-24T13:47:08,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028568 2024-11-24T13:47:08,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028568) roll requested 2024-11-24T13:47:08,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028577 2024-11-24T13:47:08,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028577 2024-11-24T13:47:08,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,582 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028577) roll requested 2024-11-24T13:47:08,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028582 2024-11-24T13:47:08,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028582 2024-11-24T13:47:08,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028582) roll requested 2024-11-24T13:47:08,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028588 2024-11-24T13:47:08,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028582 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028588 2024-11-24T13:47:08,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028588) roll requested 2024-11-24T13:47:08,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028595 2024-11-24T13:47:08,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028595 2024-11-24T13:47:08,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028595) roll requested 2024-11-24T13:47:08,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028602 2024-11-24T13:47:08,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028602 2024-11-24T13:47:08,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028602) roll requested 2024-11-24T13:47:08,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028612 2024-11-24T13:47:08,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028612 2024-11-24T13:47:08,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028612) roll requested 2024-11-24T13:47:08,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028620 2024-11-24T13:47:08,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028620 2024-11-24T13:47:08,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,630 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028620) roll requested 2024-11-24T13:47:08,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028630 2024-11-24T13:47:08,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028630 2024-11-24T13:47:08,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,638 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028630) roll requested 2024-11-24T13:47:08,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028638 2024-11-24T13:47:08,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028630 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028638 2024-11-24T13:47:08,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028638) roll requested 2024-11-24T13:47:08,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028647 2024-11-24T13:47:08,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028647 2024-11-24T13:47:08,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028647) roll requested 2024-11-24T13:47:08,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028656 2024-11-24T13:47:08,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028656 2024-11-24T13:47:08,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028656) roll requested 2024-11-24T13:47:08,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028665 2024-11-24T13:47:08,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028665 2024-11-24T13:47:08,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,673 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028665) roll requested 2024-11-24T13:47:08,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028674 2024-11-24T13:47:08,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028674 2024-11-24T13:47:08,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028674) roll requested 2024-11-24T13:47:08,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028680 2024-11-24T13:47:08,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028680 2024-11-24T13:47:08,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,686 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028680) roll requested 2024-11-24T13:47:08,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028686 2024-11-24T13:47:08,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028686 2024-11-24T13:47:08,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028686) roll requested 2024-11-24T13:47:08,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028692 2024-11-24T13:47:08,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028692 2024-11-24T13:47:08,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,698 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028692) roll requested 2024-11-24T13:47:08,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028699 2024-11-24T13:47:08,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028699 2024-11-24T13:47:08,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,706 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028699) roll requested 2024-11-24T13:47:08,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028707 2024-11-24T13:47:08,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028699 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028707 2024-11-24T13:47:08,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028707) roll requested 2024-11-24T13:47:08,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028715 2024-11-24T13:47:08,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028715 2024-11-24T13:47:08,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028715) roll requested 2024-11-24T13:47:08,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028727 2024-11-24T13:47:08,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028727 2024-11-24T13:47:08,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028727) roll requested 2024-11-24T13:47:08,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028732 2024-11-24T13:47:08,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028732 2024-11-24T13:47:08,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028732) roll requested 2024-11-24T13:47:08,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028737 2024-11-24T13:47:08,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028737 2024-11-24T13:47:08,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,743 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028737) roll requested 2024-11-24T13:47:08,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028743 2024-11-24T13:47:08,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028743 2024-11-24T13:47:08,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028743) roll requested 2024-11-24T13:47:08,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028749 2024-11-24T13:47:08,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028743 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028749 2024-11-24T13:47:08,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028749) roll requested 2024-11-24T13:47:08,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028757 2024-11-24T13:47:08,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028757 2024-11-24T13:47:08,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028757) roll requested 2024-11-24T13:47:08,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028765 2024-11-24T13:47:08,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028765 2024-11-24T13:47:08,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028765) roll requested 2024-11-24T13:47:08,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028773 2024-11-24T13:47:08,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028773 2024-11-24T13:47:08,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028773) roll requested 2024-11-24T13:47:08,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028780 2024-11-24T13:47:08,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028780 2024-11-24T13:47:08,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,787 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028780) roll requested 2024-11-24T13:47:08,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028787 2024-11-24T13:47:08,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028787 2024-11-24T13:47:08,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028787) roll requested 2024-11-24T13:47:08,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028797 2024-11-24T13:47:08,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028787 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028797 2024-11-24T13:47:08,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028797) roll requested 2024-11-24T13:47:08,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028804 2024-11-24T13:47:08,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028804 2024-11-24T13:47:08,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028804) roll requested 2024-11-24T13:47:08,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028813 2024-11-24T13:47:08,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028813 2024-11-24T13:47:08,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028813) roll requested 2024-11-24T13:47:08,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028820 2024-11-24T13:47:08,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028820 2024-11-24T13:47:08,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028820) roll requested 2024-11-24T13:47:08,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028828 2024-11-24T13:47:08,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028828 2024-11-24T13:47:08,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028828) roll requested 2024-11-24T13:47:08,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028837 2024-11-24T13:47:08,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028837 2024-11-24T13:47:08,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028837) roll requested 2024-11-24T13:47:08,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028845 2024-11-24T13:47:08,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028845 2024-11-24T13:47:08,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028845) roll requested 2024-11-24T13:47:08,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028852 2024-11-24T13:47:08,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028852 2024-11-24T13:47:08,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028852) roll requested 2024-11-24T13:47:08,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028862 2024-11-24T13:47:08,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028862 2024-11-24T13:47:08,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028862) roll requested 2024-11-24T13:47:08,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028872 2024-11-24T13:47:08,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028872 2024-11-24T13:47:08,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,879 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028872) roll requested 2024-11-24T13:47:08,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028879 2024-11-24T13:47:08,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028872 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028879 2024-11-24T13:47:08,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,886 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028879) roll requested 2024-11-24T13:47:08,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028886 2024-11-24T13:47:08,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028879 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028886 2024-11-24T13:47:08,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,891 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028886) roll requested 2024-11-24T13:47:08,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028891 2024-11-24T13:47:08,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028886 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028891 2024-11-24T13:47:08,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,897 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028891) roll requested 2024-11-24T13:47:08,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028897 2024-11-24T13:47:08,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028897 2024-11-24T13:47:08,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028897) roll requested 2024-11-24T13:47:08,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028905 2024-11-24T13:47:08,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028897 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028905 2024-11-24T13:47:08,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,913 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028905) roll requested 2024-11-24T13:47:08,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028913 2024-11-24T13:47:08,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028913 2024-11-24T13:47:08,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,922 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028913) roll requested 2024-11-24T13:47:08,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028922 2024-11-24T13:47:08,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028922 2024-11-24T13:47:08,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,930 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028922) roll requested 2024-11-24T13:47:08,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028930 2024-11-24T13:47:08,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028930 2024-11-24T13:47:08,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028930) roll requested 2024-11-24T13:47:08,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028937 2024-11-24T13:47:08,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028930 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028937 2024-11-24T13:47:08,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028937) roll requested 2024-11-24T13:47:08,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028944 2024-11-24T13:47:08,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028944 2024-11-24T13:47:08,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028944) roll requested 2024-11-24T13:47:08,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028951 2024-11-24T13:47:08,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028951 2024-11-24T13:47:08,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028951) roll requested 2024-11-24T13:47:08,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028958 2024-11-24T13:47:08,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028951 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028958 2024-11-24T13:47:08,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,967 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028958) roll requested 2024-11-24T13:47:08,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028967 2024-11-24T13:47:08,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028967 2024-11-24T13:47:08,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,975 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028967) roll requested 2024-11-24T13:47:08,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028975 2024-11-24T13:47:08,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028975 2024-11-24T13:47:08,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028975) roll requested 2024-11-24T13:47:08,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028982 2024-11-24T13:47:08,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:08,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028975 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028982 2024-11-24T13:47:08,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:08,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:08,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028982) roll requested 2024-11-24T13:47:08,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456028991 2024-11-24T13:47:09,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028991 2024-11-24T13:47:09,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456028991) roll requested 2024-11-24T13:47:09,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029002 2024-11-24T13:47:09,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456028991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029002 2024-11-24T13:47:09,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029002) roll requested 2024-11-24T13:47:09,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029010 2024-11-24T13:47:09,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029010 2024-11-24T13:47:09,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029010) roll requested 2024-11-24T13:47:09,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029016 2024-11-24T13:47:09,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029016 2024-11-24T13:47:09,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029016) roll requested 2024-11-24T13:47:09,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029022 2024-11-24T13:47:09,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029022 2024-11-24T13:47:09,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,029 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029022) roll requested 2024-11-24T13:47:09,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029030 2024-11-24T13:47:09,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029030 2024-11-24T13:47:09,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029030) roll requested 2024-11-24T13:47:09,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029037 2024-11-24T13:47:09,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029030 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029037 2024-11-24T13:47:09,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029037) roll requested 2024-11-24T13:47:09,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029045 2024-11-24T13:47:09,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029045 2024-11-24T13:47:09,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029045) roll requested 2024-11-24T13:47:09,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029052 2024-11-24T13:47:09,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029052 2024-11-24T13:47:09,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,060 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029052) roll requested 2024-11-24T13:47:09,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029060 2024-11-24T13:47:09,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029060 2024-11-24T13:47:09,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,067 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029060) roll requested 2024-11-24T13:47:09,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029067 2024-11-24T13:47:09,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029060 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029067 2024-11-24T13:47:09,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029067) roll requested 2024-11-24T13:47:09,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029077 2024-11-24T13:47:09,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029067 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029077 2024-11-24T13:47:09,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029077) roll requested 2024-11-24T13:47:09,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029086 2024-11-24T13:47:09,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029086 2024-11-24T13:47:09,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029086) roll requested 2024-11-24T13:47:09,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029099 2024-11-24T13:47:09,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029099 2024-11-24T13:47:09,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029099) roll requested 2024-11-24T13:47:09,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029107 2024-11-24T13:47:09,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029107 2024-11-24T13:47:09,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029107) roll requested 2024-11-24T13:47:09,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029115 2024-11-24T13:47:09,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029115 2024-11-24T13:47:09,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029115) roll requested 2024-11-24T13:47:09,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029123 2024-11-24T13:47:09,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029115 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029123 2024-11-24T13:47:09,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029123) roll requested 2024-11-24T13:47:09,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029130 2024-11-24T13:47:09,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029130 2024-11-24T13:47:09,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029130) roll requested 2024-11-24T13:47:09,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029141 2024-11-24T13:47:09,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029141 2024-11-24T13:47:09,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029141) roll requested 2024-11-24T13:47:09,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029148 2024-11-24T13:47:09,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029148 2024-11-24T13:47:09,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,154 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029148) roll requested 2024-11-24T13:47:09,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029154 2024-11-24T13:47:09,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029154 2024-11-24T13:47:09,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029154) roll requested 2024-11-24T13:47:09,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029159 2024-11-24T13:47:09,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029154 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029159 2024-11-24T13:47:09,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029159) roll requested 2024-11-24T13:47:09,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029165 2024-11-24T13:47:09,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029165 2024-11-24T13:47:09,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029165) roll requested 2024-11-24T13:47:09,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029172 2024-11-24T13:47:09,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029172 2024-11-24T13:47:09,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029172) roll requested 2024-11-24T13:47:09,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029180 2024-11-24T13:47:09,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029180 2024-11-24T13:47:09,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029180) roll requested 2024-11-24T13:47:09,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029188 2024-11-24T13:47:09,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029188 2024-11-24T13:47:09,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029188) roll requested 2024-11-24T13:47:09,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029201 2024-11-24T13:47:09,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029201 2024-11-24T13:47:09,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029201) roll requested 2024-11-24T13:47:09,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029215 2024-11-24T13:47:09,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029215 2024-11-24T13:47:09,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029215) roll requested 2024-11-24T13:47:09,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029227 2024-11-24T13:47:09,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029227 2024-11-24T13:47:09,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,234 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029227) roll requested 2024-11-24T13:47:09,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029235 2024-11-24T13:47:09,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029235 2024-11-24T13:47:09,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,243 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029235) roll requested 2024-11-24T13:47:09,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029243 2024-11-24T13:47:09,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029235 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029243 2024-11-24T13:47:09,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029243) roll requested 2024-11-24T13:47:09,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029255 2024-11-24T13:47:09,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029243 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029255 2024-11-24T13:47:09,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029255) roll requested 2024-11-24T13:47:09,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029264 2024-11-24T13:47:09,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029264 2024-11-24T13:47:09,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029264) roll requested 2024-11-24T13:47:09,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029271 2024-11-24T13:47:09,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029271 2024-11-24T13:47:09,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029271) roll requested 2024-11-24T13:47:09,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029279 2024-11-24T13:47:09,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029279 2024-11-24T13:47:09,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029279) roll requested 2024-11-24T13:47:09,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029285 2024-11-24T13:47:09,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029285 2024-11-24T13:47:09,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029285) roll requested 2024-11-24T13:47:09,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029299 2024-11-24T13:47:09,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029299 2024-11-24T13:47:09,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,307 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029299) roll requested 2024-11-24T13:47:09,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029307 2024-11-24T13:47:09,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029307 2024-11-24T13:47:09,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,315 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029307) roll requested 2024-11-24T13:47:09,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029315 2024-11-24T13:47:09,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029315 2024-11-24T13:47:09,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029315) roll requested 2024-11-24T13:47:09,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029323 2024-11-24T13:47:09,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029315 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029323 2024-11-24T13:47:09,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029323) roll requested 2024-11-24T13:47:09,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029332 2024-11-24T13:47:09,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029332 2024-11-24T13:47:09,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029332) roll requested 2024-11-24T13:47:09,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029340 2024-11-24T13:47:09,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029340 2024-11-24T13:47:09,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029340) roll requested 2024-11-24T13:47:09,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029346 2024-11-24T13:47:09,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029346 2024-11-24T13:47:09,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029346) roll requested 2024-11-24T13:47:09,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029352 2024-11-24T13:47:09,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029352 2024-11-24T13:47:09,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,357 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029352) roll requested 2024-11-24T13:47:09,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029357 2024-11-24T13:47:09,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029357 2024-11-24T13:47:09,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029357) roll requested 2024-11-24T13:47:09,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029363 2024-11-24T13:47:09,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029363 2024-11-24T13:47:09,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029363) roll requested 2024-11-24T13:47:09,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029368 2024-11-24T13:47:09,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029368 2024-11-24T13:47:09,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029368) roll requested 2024-11-24T13:47:09,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029374 2024-11-24T13:47:09,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029374 2024-11-24T13:47:09,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029374) roll requested 2024-11-24T13:47:09,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029380 2024-11-24T13:47:09,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029380 2024-11-24T13:47:09,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029380) roll requested 2024-11-24T13:47:09,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029387 2024-11-24T13:47:09,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029387 2024-11-24T13:47:09,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029387) roll requested 2024-11-24T13:47:09,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029392 2024-11-24T13:47:09,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029392 2024-11-24T13:47:09,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029392) roll requested 2024-11-24T13:47:09,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029397 2024-11-24T13:47:09,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029397 2024-11-24T13:47:09,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,402 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029397) roll requested 2024-11-24T13:47:09,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029402 2024-11-24T13:47:09,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029402 2024-11-24T13:47:09,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,407 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029402) roll requested 2024-11-24T13:47:09,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029407 2024-11-24T13:47:09,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029402 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029407 2024-11-24T13:47:09,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029407) roll requested 2024-11-24T13:47:09,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029413 2024-11-24T13:47:09,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029413 2024-11-24T13:47:09,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,419 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029413) roll requested 2024-11-24T13:47:09,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029420 2024-11-24T13:47:09,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029420 2024-11-24T13:47:09,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029420) roll requested 2024-11-24T13:47:09,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029427 2024-11-24T13:47:09,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029427 2024-11-24T13:47:09,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029427) roll requested 2024-11-24T13:47:09,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029435 2024-11-24T13:47:09,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029435 2024-11-24T13:47:09,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,444 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029435) roll requested 2024-11-24T13:47:09,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029444 2024-11-24T13:47:09,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029444 2024-11-24T13:47:09,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029444) roll requested 2024-11-24T13:47:09,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029450 2024-11-24T13:47:09,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029450 2024-11-24T13:47:09,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029450) roll requested 2024-11-24T13:47:09,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029455 2024-11-24T13:47:09,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029455 2024-11-24T13:47:09,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,463 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029455) roll requested 2024-11-24T13:47:09,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029463 2024-11-24T13:47:09,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029455 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029463 2024-11-24T13:47:09,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029463) roll requested 2024-11-24T13:47:09,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029472 2024-11-24T13:47:09,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029472 2024-11-24T13:47:09,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029472) roll requested 2024-11-24T13:47:09,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029481 2024-11-24T13:47:09,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029481 2024-11-24T13:47:09,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029481) roll requested 2024-11-24T13:47:09,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029491 2024-11-24T13:47:09,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029491 2024-11-24T13:47:09,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029491) roll requested 2024-11-24T13:47:09,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029501 2024-11-24T13:47:09,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029501 2024-11-24T13:47:09,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029501) roll requested 2024-11-24T13:47:09,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029506 2024-11-24T13:47:09,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029506 2024-11-24T13:47:09,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,511 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029506) roll requested 2024-11-24T13:47:09,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029511 2024-11-24T13:47:09,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029511 2024-11-24T13:47:09,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029511) roll requested 2024-11-24T13:47:09,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029516 2024-11-24T13:47:09,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029516 2024-11-24T13:47:09,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029516) roll requested 2024-11-24T13:47:09,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029522 2024-11-24T13:47:09,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029516 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029522 2024-11-24T13:47:09,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029522) roll requested 2024-11-24T13:47:09,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029528 2024-11-24T13:47:09,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029528 2024-11-24T13:47:09,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029528) roll requested 2024-11-24T13:47:09,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029535 2024-11-24T13:47:09,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029535 2024-11-24T13:47:09,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029535) roll requested 2024-11-24T13:47:09,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029543 2024-11-24T13:47:09,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029543 2024-11-24T13:47:09,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029543) roll requested 2024-11-24T13:47:09,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029550 2024-11-24T13:47:09,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029550 2024-11-24T13:47:09,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029550) roll requested 2024-11-24T13:47:09,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029558 2024-11-24T13:47:09,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029550 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029558 2024-11-24T13:47:09,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029558) roll requested 2024-11-24T13:47:09,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029566 2024-11-24T13:47:09,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029566 2024-11-24T13:47:09,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,573 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029566) roll requested 2024-11-24T13:47:09,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029573 2024-11-24T13:47:09,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029573 2024-11-24T13:47:09,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,579 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029573) roll requested 2024-11-24T13:47:09,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029579 2024-11-24T13:47:09,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029573 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029579 2024-11-24T13:47:09,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029579) roll requested 2024-11-24T13:47:09,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029585 2024-11-24T13:47:09,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029579 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029585 2024-11-24T13:47:09,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029585) roll requested 2024-11-24T13:47:09,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029592 2024-11-24T13:47:09,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029592 2024-11-24T13:47:09,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,600 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029592) roll requested 2024-11-24T13:47:09,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029601 2024-11-24T13:47:09,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029601 2024-11-24T13:47:09,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029601) roll requested 2024-11-24T13:47:09,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029610 2024-11-24T13:47:09,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029610 2024-11-24T13:47:09,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029610) roll requested 2024-11-24T13:47:09,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029621 2024-11-24T13:47:09,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029610 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029621 2024-11-24T13:47:09,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029621) roll requested 2024-11-24T13:47:09,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029628 2024-11-24T13:47:09,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029628 2024-11-24T13:47:09,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029628) roll requested 2024-11-24T13:47:09,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029636 2024-11-24T13:47:09,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029636 2024-11-24T13:47:09,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029636) roll requested 2024-11-24T13:47:09,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029643 2024-11-24T13:47:09,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029643 2024-11-24T13:47:09,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029643) roll requested 2024-11-24T13:47:09,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029648 2024-11-24T13:47:09,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029643 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029648 2024-11-24T13:47:09,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029648) roll requested 2024-11-24T13:47:09,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029654 2024-11-24T13:47:09,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029654 2024-11-24T13:47:09,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029654) roll requested 2024-11-24T13:47:09,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029661 2024-11-24T13:47:09,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029661 2024-11-24T13:47:09,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029661) roll requested 2024-11-24T13:47:09,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029669 2024-11-24T13:47:09,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029669 2024-11-24T13:47:09,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029669) roll requested 2024-11-24T13:47:09,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029676 2024-11-24T13:47:09,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029676 2024-11-24T13:47:09,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029676) roll requested 2024-11-24T13:47:09,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029684 2024-11-24T13:47:09,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029684 2024-11-24T13:47:09,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029684) roll requested 2024-11-24T13:47:09,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029692 2024-11-24T13:47:09,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029692 2024-11-24T13:47:09,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029692) roll requested 2024-11-24T13:47:09,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029701 2024-11-24T13:47:09,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029701 2024-11-24T13:47:09,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029701) roll requested 2024-11-24T13:47:09,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029709 2024-11-24T13:47:09,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029709 2024-11-24T13:47:09,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029709) roll requested 2024-11-24T13:47:09,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029719 2024-11-24T13:47:09,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029719 2024-11-24T13:47:09,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029719) roll requested 2024-11-24T13:47:09,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029728 2024-11-24T13:47:09,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029728 2024-11-24T13:47:09,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,738 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029728) roll requested 2024-11-24T13:47:09,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029738 2024-11-24T13:47:09,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029738 2024-11-24T13:47:09,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029738) roll requested 2024-11-24T13:47:09,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029747 2024-11-24T13:47:09,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029738 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029747 2024-11-24T13:47:09,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,754 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029747) roll requested 2024-11-24T13:47:09,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029754 2024-11-24T13:47:09,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029754 2024-11-24T13:47:09,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,762 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029754) roll requested 2024-11-24T13:47:09,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029762 2024-11-24T13:47:09,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029754 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029762 2024-11-24T13:47:09,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029762) roll requested 2024-11-24T13:47:09,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029770 2024-11-24T13:47:09,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029762 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029770 2024-11-24T13:47:09,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,777 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029770) roll requested 2024-11-24T13:47:09,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029778 2024-11-24T13:47:09,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029778 2024-11-24T13:47:09,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029778) roll requested 2024-11-24T13:47:09,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029785 2024-11-24T13:47:09,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029785 2024-11-24T13:47:09,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029785) roll requested 2024-11-24T13:47:09,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029792 2024-11-24T13:47:09,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029792 2024-11-24T13:47:09,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029792) roll requested 2024-11-24T13:47:09,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029799 2024-11-24T13:47:09,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029792 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029799 2024-11-24T13:47:09,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029799) roll requested 2024-11-24T13:47:09,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029806 2024-11-24T13:47:09,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029806 2024-11-24T13:47:09,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029806) roll requested 2024-11-24T13:47:09,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029814 2024-11-24T13:47:09,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029814 2024-11-24T13:47:09,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029814) roll requested 2024-11-24T13:47:09,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029822 2024-11-24T13:47:09,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029822 2024-11-24T13:47:09,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029822) roll requested 2024-11-24T13:47:09,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029833 2024-11-24T13:47:09,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029833 2024-11-24T13:47:09,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,842 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029833) roll requested 2024-11-24T13:47:09,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029842 2024-11-24T13:47:09,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029842 2024-11-24T13:47:09,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029842) roll requested 2024-11-24T13:47:09,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029848 2024-11-24T13:47:09,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029848 2024-11-24T13:47:09,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029848) roll requested 2024-11-24T13:47:09,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029853 2024-11-24T13:47:09,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029853 2024-11-24T13:47:09,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029853) roll requested 2024-11-24T13:47:09,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029858 2024-11-24T13:47:09,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029858 2024-11-24T13:47:09,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029858) roll requested 2024-11-24T13:47:09,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029864 2024-11-24T13:47:09,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029864 2024-11-24T13:47:09,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029864) roll requested 2024-11-24T13:47:09,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029872 2024-11-24T13:47:09,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029872 2024-11-24T13:47:09,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029872) roll requested 2024-11-24T13:47:09,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029881 2024-11-24T13:47:09,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029872 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029881 2024-11-24T13:47:09,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029881) roll requested 2024-11-24T13:47:09,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029888 2024-11-24T13:47:09,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029888 2024-11-24T13:47:09,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029888) roll requested 2024-11-24T13:47:09,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029895 2024-11-24T13:47:09,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029895 2024-11-24T13:47:09,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029895) roll requested 2024-11-24T13:47:09,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029903 2024-11-24T13:47:09,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029903 2024-11-24T13:47:09,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,913 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029903) roll requested 2024-11-24T13:47:09,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029913 2024-11-24T13:47:09,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029913 2024-11-24T13:47:09,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029913) roll requested 2024-11-24T13:47:09,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029921 2024-11-24T13:47:09,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029921 2024-11-24T13:47:09,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,930 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029921) roll requested 2024-11-24T13:47:09,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029930 2024-11-24T13:47:09,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029930 2024-11-24T13:47:09,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,938 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029930) roll requested 2024-11-24T13:47:09,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029939 2024-11-24T13:47:09,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029930 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029939 2024-11-24T13:47:09,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029939) roll requested 2024-11-24T13:47:09,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029948 2024-11-24T13:47:09,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029948 2024-11-24T13:47:09,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029948) roll requested 2024-11-24T13:47:09,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029956 2024-11-24T13:47:09,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029948 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029956 2024-11-24T13:47:09,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,963 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029956) roll requested 2024-11-24T13:47:09,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029963 2024-11-24T13:47:09,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029963 2024-11-24T13:47:09,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,970 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029963) roll requested 2024-11-24T13:47:09,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029970 2024-11-24T13:47:09,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029963 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029970 2024-11-24T13:47:09,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,978 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029970) roll requested 2024-11-24T13:47:09,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029978 2024-11-24T13:47:09,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029970 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029978 2024-11-24T13:47:09,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029978) roll requested 2024-11-24T13:47:09,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029986 2024-11-24T13:47:09,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:09,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029978 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029986 2024-11-24T13:47:09,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:09,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:09,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029986) roll requested 2024-11-24T13:47:09,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456029995 2024-11-24T13:47:10,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029995 2024-11-24T13:47:10,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456029995) roll requested 2024-11-24T13:47:10,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030004 2024-11-24T13:47:10,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456029995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030004 2024-11-24T13:47:10,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030004) roll requested 2024-11-24T13:47:10,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030011 2024-11-24T13:47:10,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030011 2024-11-24T13:47:10,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030011) roll requested 2024-11-24T13:47:10,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030019 2024-11-24T13:47:10,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030019 2024-11-24T13:47:10,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,027 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030019) roll requested 2024-11-24T13:47:10,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030028 2024-11-24T13:47:10,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030028 2024-11-24T13:47:10,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030028) roll requested 2024-11-24T13:47:10,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030035 2024-11-24T13:47:10,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030028 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030035 2024-11-24T13:47:10,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030035) roll requested 2024-11-24T13:47:10,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030045 2024-11-24T13:47:10,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030045 2024-11-24T13:47:10,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030045) roll requested 2024-11-24T13:47:10,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030052 2024-11-24T13:47:10,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030052 2024-11-24T13:47:10,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,060 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030052) roll requested 2024-11-24T13:47:10,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030060 2024-11-24T13:47:10,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030060 2024-11-24T13:47:10,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030060) roll requested 2024-11-24T13:47:10,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030069 2024-11-24T13:47:10,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030060 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030069 2024-11-24T13:47:10,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030069) roll requested 2024-11-24T13:47:10,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030078 2024-11-24T13:47:10,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030078 2024-11-24T13:47:10,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030078) roll requested 2024-11-24T13:47:10,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030086 2024-11-24T13:47:10,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030078 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030086 2024-11-24T13:47:10,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,095 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030086) roll requested 2024-11-24T13:47:10,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030096 2024-11-24T13:47:10,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030096 2024-11-24T13:47:10,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030096) roll requested 2024-11-24T13:47:10,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030105 2024-11-24T13:47:10,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030105 2024-11-24T13:47:10,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030105) roll requested 2024-11-24T13:47:10,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030114 2024-11-24T13:47:10,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030114 2024-11-24T13:47:10,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030114) roll requested 2024-11-24T13:47:10,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030122 2024-11-24T13:47:10,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030122 2024-11-24T13:47:10,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030122) roll requested 2024-11-24T13:47:10,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030131 2024-11-24T13:47:10,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030131 2024-11-24T13:47:10,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030131) roll requested 2024-11-24T13:47:10,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030142 2024-11-24T13:47:10,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030142 2024-11-24T13:47:10,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030142) roll requested 2024-11-24T13:47:10,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030150 2024-11-24T13:47:10,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030150 2024-11-24T13:47:10,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030150) roll requested 2024-11-24T13:47:10,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030157 2024-11-24T13:47:10,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030157 2024-11-24T13:47:10,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030157) roll requested 2024-11-24T13:47:10,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030165 2024-11-24T13:47:10,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030165 2024-11-24T13:47:10,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,174 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030165) roll requested 2024-11-24T13:47:10,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030175 2024-11-24T13:47:10,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030175 2024-11-24T13:47:10,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030175) roll requested 2024-11-24T13:47:10,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030182 2024-11-24T13:47:10,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030182 2024-11-24T13:47:10,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030182) roll requested 2024-11-24T13:47:10,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030189 2024-11-24T13:47:10,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030182 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030189 2024-11-24T13:47:10,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030189) roll requested 2024-11-24T13:47:10,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030197 2024-11-24T13:47:10,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030197 2024-11-24T13:47:10,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,207 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030197) roll requested 2024-11-24T13:47:10,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030207 2024-11-24T13:47:10,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030207 2024-11-24T13:47:10,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030207) roll requested 2024-11-24T13:47:10,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030220 2024-11-24T13:47:10,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030220 2024-11-24T13:47:10,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030220) roll requested 2024-11-24T13:47:10,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030226 2024-11-24T13:47:10,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030226 2024-11-24T13:47:10,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030226) roll requested 2024-11-24T13:47:10,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030233 2024-11-24T13:47:10,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030233 2024-11-24T13:47:10,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030233) roll requested 2024-11-24T13:47:10,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030238 2024-11-24T13:47:10,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030238 2024-11-24T13:47:10,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030238) roll requested 2024-11-24T13:47:10,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030248 2024-11-24T13:47:10,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030248 2024-11-24T13:47:10,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030248) roll requested 2024-11-24T13:47:10,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030253 2024-11-24T13:47:10,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030253 2024-11-24T13:47:10,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,258 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030253) roll requested 2024-11-24T13:47:10,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030258 2024-11-24T13:47:10,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030258 2024-11-24T13:47:10,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030258) roll requested 2024-11-24T13:47:10,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030263 2024-11-24T13:47:10,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030258 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030263 2024-11-24T13:47:10,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,269 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030263) roll requested 2024-11-24T13:47:10,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030269 2024-11-24T13:47:10,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030269 2024-11-24T13:47:10,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030269) roll requested 2024-11-24T13:47:10,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030274 2024-11-24T13:47:10,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030274 2024-11-24T13:47:10,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030274) roll requested 2024-11-24T13:47:10,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030280 2024-11-24T13:47:10,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030280 2024-11-24T13:47:10,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030280) roll requested 2024-11-24T13:47:10,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030285 2024-11-24T13:47:10,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030285 2024-11-24T13:47:10,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030285) roll requested 2024-11-24T13:47:10,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030292 2024-11-24T13:47:10,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030292 2024-11-24T13:47:10,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030292) roll requested 2024-11-24T13:47:10,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030298 2024-11-24T13:47:10,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030298 2024-11-24T13:47:10,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,303 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030298) roll requested 2024-11-24T13:47:10,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030303 2024-11-24T13:47:10,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030303 2024-11-24T13:47:10,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,309 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030303) roll requested 2024-11-24T13:47:10,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030309 2024-11-24T13:47:10,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030303 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030309 2024-11-24T13:47:10,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030309) roll requested 2024-11-24T13:47:10,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030314 2024-11-24T13:47:10,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030314 2024-11-24T13:47:10,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030314) roll requested 2024-11-24T13:47:10,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030319 2024-11-24T13:47:10,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030319 2024-11-24T13:47:10,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030319) roll requested 2024-11-24T13:47:10,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030325 2024-11-24T13:47:10,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030325 2024-11-24T13:47:10,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030325) roll requested 2024-11-24T13:47:10,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030331 2024-11-24T13:47:10,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030331 2024-11-24T13:47:10,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030331) roll requested 2024-11-24T13:47:10,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030336 2024-11-24T13:47:10,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030336 2024-11-24T13:47:10,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030336) roll requested 2024-11-24T13:47:10,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030341 2024-11-24T13:47:10,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030341 2024-11-24T13:47:10,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030341) roll requested 2024-11-24T13:47:10,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030346 2024-11-24T13:47:10,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030346 2024-11-24T13:47:10,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030346) roll requested 2024-11-24T13:47:10,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030352 2024-11-24T13:47:10,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030352 2024-11-24T13:47:10,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,357 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030352) roll requested 2024-11-24T13:47:10,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030357 2024-11-24T13:47:10,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030357 2024-11-24T13:47:10,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030357) roll requested 2024-11-24T13:47:10,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030363 2024-11-24T13:47:10,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030363 2024-11-24T13:47:10,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030363) roll requested 2024-11-24T13:47:10,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030368 2024-11-24T13:47:10,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030368 2024-11-24T13:47:10,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,373 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030368) roll requested 2024-11-24T13:47:10,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030373 2024-11-24T13:47:10,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030373 2024-11-24T13:47:10,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030373) roll requested 2024-11-24T13:47:10,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030379 2024-11-24T13:47:10,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030373 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030379 2024-11-24T13:47:10,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030379) roll requested 2024-11-24T13:47:10,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030387 2024-11-24T13:47:10,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030387 2024-11-24T13:47:10,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,391 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030387) roll requested 2024-11-24T13:47:10,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030392 2024-11-24T13:47:10,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030392 2024-11-24T13:47:10,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030392) roll requested 2024-11-24T13:47:10,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030397 2024-11-24T13:47:10,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030397 2024-11-24T13:47:10,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,401 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030397) roll requested 2024-11-24T13:47:10,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030401 2024-11-24T13:47:10,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030401 2024-11-24T13:47:10,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030401) roll requested 2024-11-24T13:47:10,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030406 2024-11-24T13:47:10,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030406 2024-11-24T13:47:10,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030406) roll requested 2024-11-24T13:47:10,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030411 2024-11-24T13:47:10,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030411 2024-11-24T13:47:10,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030411) roll requested 2024-11-24T13:47:10,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030416 2024-11-24T13:47:10,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030416 2024-11-24T13:47:10,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030416) roll requested 2024-11-24T13:47:10,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030423 2024-11-24T13:47:10,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030423 2024-11-24T13:47:10,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030423) roll requested 2024-11-24T13:47:10,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030429 2024-11-24T13:47:10,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030423 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030429 2024-11-24T13:47:10,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030429) roll requested 2024-11-24T13:47:10,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030436 2024-11-24T13:47:10,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030436 2024-11-24T13:47:10,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030436) roll requested 2024-11-24T13:47:10,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030445 2024-11-24T13:47:10,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030445 2024-11-24T13:47:10,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,454 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030445) roll requested 2024-11-24T13:47:10,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030455 2024-11-24T13:47:10,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030455 2024-11-24T13:47:10,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030455) roll requested 2024-11-24T13:47:10,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030465 2024-11-24T13:47:10,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030455 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030465 2024-11-24T13:47:10,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030465) roll requested 2024-11-24T13:47:10,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030471 2024-11-24T13:47:10,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030471 2024-11-24T13:47:10,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030471) roll requested 2024-11-24T13:47:10,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030476 2024-11-24T13:47:10,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030476 2024-11-24T13:47:10,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030476) roll requested 2024-11-24T13:47:10,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030483 2024-11-24T13:47:10,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030483 2024-11-24T13:47:10,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030483) roll requested 2024-11-24T13:47:10,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030488 2024-11-24T13:47:10,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030488 2024-11-24T13:47:10,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030488) roll requested 2024-11-24T13:47:10,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030497 2024-11-24T13:47:10,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030497 2024-11-24T13:47:10,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030497) roll requested 2024-11-24T13:47:10,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030502 2024-11-24T13:47:10,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030502 2024-11-24T13:47:10,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030502) roll requested 2024-11-24T13:47:10,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030507 2024-11-24T13:47:10,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030507 2024-11-24T13:47:10,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,511 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030507) roll requested 2024-11-24T13:47:10,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030512 2024-11-24T13:47:10,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030512 2024-11-24T13:47:10,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030512) roll requested 2024-11-24T13:47:10,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030517 2024-11-24T13:47:10,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030517 2024-11-24T13:47:10,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030517) roll requested 2024-11-24T13:47:10,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030523 2024-11-24T13:47:10,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030523 2024-11-24T13:47:10,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030523) roll requested 2024-11-24T13:47:10,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030528 2024-11-24T13:47:10,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030523 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030528 2024-11-24T13:47:10,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030528) roll requested 2024-11-24T13:47:10,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030533 2024-11-24T13:47:10,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030533 2024-11-24T13:47:10,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,538 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030533) roll requested 2024-11-24T13:47:10,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030538 2024-11-24T13:47:10,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030538 2024-11-24T13:47:10,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030538) roll requested 2024-11-24T13:47:10,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030543 2024-11-24T13:47:10,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030538 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030543 2024-11-24T13:47:10,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030543) roll requested 2024-11-24T13:47:10,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030549 2024-11-24T13:47:10,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030549 2024-11-24T13:47:10,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,554 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030549) roll requested 2024-11-24T13:47:10,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030554 2024-11-24T13:47:10,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030554 2024-11-24T13:47:10,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030554) roll requested 2024-11-24T13:47:10,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030560 2024-11-24T13:47:10,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030554 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030560 2024-11-24T13:47:10,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030560) roll requested 2024-11-24T13:47:10,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030565 2024-11-24T13:47:10,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030565 2024-11-24T13:47:10,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030565) roll requested 2024-11-24T13:47:10,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030578 2024-11-24T13:47:10,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030578 2024-11-24T13:47:10,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030578) roll requested 2024-11-24T13:47:10,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030584 2024-11-24T13:47:10,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030584 2024-11-24T13:47:10,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030584) roll requested 2024-11-24T13:47:10,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030589 2024-11-24T13:47:10,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030589 2024-11-24T13:47:10,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030589) roll requested 2024-11-24T13:47:10,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030594 2024-11-24T13:47:10,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030594 2024-11-24T13:47:10,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030594) roll requested 2024-11-24T13:47:10,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030599 2024-11-24T13:47:10,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030599 2024-11-24T13:47:10,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030599) roll requested 2024-11-24T13:47:10,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030605 2024-11-24T13:47:10,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030605 2024-11-24T13:47:10,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030605) roll requested 2024-11-24T13:47:10,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030612 2024-11-24T13:47:10,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030612 2024-11-24T13:47:10,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030612) roll requested 2024-11-24T13:47:10,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030617 2024-11-24T13:47:10,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030617 2024-11-24T13:47:10,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030617) roll requested 2024-11-24T13:47:10,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030623 2024-11-24T13:47:10,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030623 2024-11-24T13:47:10,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030623) roll requested 2024-11-24T13:47:10,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030632 2024-11-24T13:47:10,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030632 2024-11-24T13:47:10,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030632) roll requested 2024-11-24T13:47:10,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030639 2024-11-24T13:47:10,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030639 2024-11-24T13:47:10,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030639) roll requested 2024-11-24T13:47:10,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030647 2024-11-24T13:47:10,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030647 2024-11-24T13:47:10,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030647) roll requested 2024-11-24T13:47:10,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030652 2024-11-24T13:47:10,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030652 2024-11-24T13:47:10,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030652) roll requested 2024-11-24T13:47:10,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030657 2024-11-24T13:47:10,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030657 2024-11-24T13:47:10,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030657) roll requested 2024-11-24T13:47:10,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030662 2024-11-24T13:47:10,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030662 2024-11-24T13:47:10,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030662) roll requested 2024-11-24T13:47:10,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030668 2024-11-24T13:47:10,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030668 2024-11-24T13:47:10,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,673 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030668) roll requested 2024-11-24T13:47:10,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030673 2024-11-24T13:47:10,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030673 2024-11-24T13:47:10,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,678 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030673) roll requested 2024-11-24T13:47:10,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030679 2024-11-24T13:47:10,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030679 2024-11-24T13:47:10,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030679) roll requested 2024-11-24T13:47:10,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030684 2024-11-24T13:47:10,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030684 2024-11-24T13:47:10,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030684) roll requested 2024-11-24T13:47:10,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030689 2024-11-24T13:47:10,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030689 2024-11-24T13:47:10,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030689) roll requested 2024-11-24T13:47:10,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030695 2024-11-24T13:47:10,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030695 2024-11-24T13:47:10,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030695) roll requested 2024-11-24T13:47:10,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030701 2024-11-24T13:47:10,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030701 2024-11-24T13:47:10,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,706 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030701) roll requested 2024-11-24T13:47:10,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030706 2024-11-24T13:47:10,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030706 2024-11-24T13:47:10,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030706) roll requested 2024-11-24T13:47:10,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030712 2024-11-24T13:47:10,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030706 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030712 2024-11-24T13:47:10,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030712) roll requested 2024-11-24T13:47:10,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030717 2024-11-24T13:47:10,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030717 2024-11-24T13:47:10,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030717) roll requested 2024-11-24T13:47:10,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030727 2024-11-24T13:47:10,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030727 2024-11-24T13:47:10,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030727) roll requested 2024-11-24T13:47:10,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030735 2024-11-24T13:47:10,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030735 2024-11-24T13:47:10,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,741 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030735) roll requested 2024-11-24T13:47:10,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030742 2024-11-24T13:47:10,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030742 2024-11-24T13:47:10,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030742) roll requested 2024-11-24T13:47:10,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030749 2024-11-24T13:47:10,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030749 2024-11-24T13:47:10,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030749) roll requested 2024-11-24T13:47:10,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030758 2024-11-24T13:47:10,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030758 2024-11-24T13:47:10,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030758) roll requested 2024-11-24T13:47:10,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030768 2024-11-24T13:47:10,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030768 2024-11-24T13:47:10,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030768) roll requested 2024-11-24T13:47:10,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030777 2024-11-24T13:47:10,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030777 2024-11-24T13:47:10,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030777) roll requested 2024-11-24T13:47:10,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030783 2024-11-24T13:47:10,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030783 2024-11-24T13:47:10,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030783) roll requested 2024-11-24T13:47:10,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030791 2024-11-24T13:47:10,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030791 2024-11-24T13:47:10,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030791) roll requested 2024-11-24T13:47:10,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030796 2024-11-24T13:47:10,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030796 2024-11-24T13:47:10,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030796) roll requested 2024-11-24T13:47:10,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030803 2024-11-24T13:47:10,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030803 2024-11-24T13:47:10,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030803) roll requested 2024-11-24T13:47:10,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030808 2024-11-24T13:47:10,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030808 2024-11-24T13:47:10,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030808) roll requested 2024-11-24T13:47:10,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030817 2024-11-24T13:47:10,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030817 2024-11-24T13:47:10,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,823 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030817) roll requested 2024-11-24T13:47:10,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030823 2024-11-24T13:47:10,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030823 2024-11-24T13:47:10,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,830 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030823) roll requested 2024-11-24T13:47:10,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030831 2024-11-24T13:47:10,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030823 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030831 2024-11-24T13:47:10,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030831) roll requested 2024-11-24T13:47:10,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030836 2024-11-24T13:47:10,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030836 2024-11-24T13:47:10,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,842 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030836) roll requested 2024-11-24T13:47:10,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030842 2024-11-24T13:47:10,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030842 2024-11-24T13:47:10,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030842) roll requested 2024-11-24T13:47:10,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030847 2024-11-24T13:47:10,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030847 2024-11-24T13:47:10,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030847) roll requested 2024-11-24T13:47:10,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030852 2024-11-24T13:47:10,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030852 2024-11-24T13:47:10,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030852) roll requested 2024-11-24T13:47:10,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030857 2024-11-24T13:47:10,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030857 2024-11-24T13:47:10,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030857) roll requested 2024-11-24T13:47:10,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030862 2024-11-24T13:47:10,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030862 2024-11-24T13:47:10,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030862) roll requested 2024-11-24T13:47:10,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030868 2024-11-24T13:47:10,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030868 2024-11-24T13:47:10,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030868) roll requested 2024-11-24T13:47:10,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030873 2024-11-24T13:47:10,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030873 2024-11-24T13:47:10,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030873) roll requested 2024-11-24T13:47:10,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030878 2024-11-24T13:47:10,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030873 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030878 2024-11-24T13:47:10,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030878) roll requested 2024-11-24T13:47:10,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030884 2024-11-24T13:47:10,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030884 2024-11-24T13:47:10,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030884) roll requested 2024-11-24T13:47:10,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030889 2024-11-24T13:47:10,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030889 2024-11-24T13:47:10,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030889) roll requested 2024-11-24T13:47:10,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030894 2024-11-24T13:47:10,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030894 2024-11-24T13:47:10,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030894) roll requested 2024-11-24T13:47:10,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030900 2024-11-24T13:47:10,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030900 2024-11-24T13:47:10,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030900) roll requested 2024-11-24T13:47:10,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030905 2024-11-24T13:47:10,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030905 2024-11-24T13:47:10,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030905) roll requested 2024-11-24T13:47:10,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030911 2024-11-24T13:47:10,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030911 2024-11-24T13:47:10,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030911) roll requested 2024-11-24T13:47:10,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030919 2024-11-24T13:47:10,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030919 2024-11-24T13:47:10,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030919) roll requested 2024-11-24T13:47:10,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030925 2024-11-24T13:47:10,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030925 2024-11-24T13:47:10,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,930 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030925) roll requested 2024-11-24T13:47:10,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030931 2024-11-24T13:47:10,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030931 2024-11-24T13:47:10,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030931) roll requested 2024-11-24T13:47:10,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030936 2024-11-24T13:47:10,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030936 2024-11-24T13:47:10,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030936) roll requested 2024-11-24T13:47:10,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030943 2024-11-24T13:47:10,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030943 2024-11-24T13:47:10,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,950 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030943) roll requested 2024-11-24T13:47:10,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030951 2024-11-24T13:47:10,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030943 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030951 2024-11-24T13:47:10,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030951) roll requested 2024-11-24T13:47:10,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030958 2024-11-24T13:47:10,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030951 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030958 2024-11-24T13:47:10,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030958) roll requested 2024-11-24T13:47:10,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030966 2024-11-24T13:47:10,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030966 2024-11-24T13:47:10,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030966) roll requested 2024-11-24T13:47:10,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030971 2024-11-24T13:47:10,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030971 2024-11-24T13:47:10,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030971) roll requested 2024-11-24T13:47:10,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030979 2024-11-24T13:47:10,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030979 2024-11-24T13:47:10,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030979) roll requested 2024-11-24T13:47:10,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030987 2024-11-24T13:47:10,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030979 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030987 2024-11-24T13:47:10,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030987) roll requested 2024-11-24T13:47:10,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030992 2024-11-24T13:47:10,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:10,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030992 2024-11-24T13:47:10,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:10,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:10,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030992) roll requested 2024-11-24T13:47:10,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456030999 2024-11-24T13:47:11,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030999 2024-11-24T13:47:11,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456030999) roll requested 2024-11-24T13:47:11,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031004 2024-11-24T13:47:11,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456030999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031004 2024-11-24T13:47:11,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031004) roll requested 2024-11-24T13:47:11,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031009 2024-11-24T13:47:11,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031009 2024-11-24T13:47:11,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031009) roll requested 2024-11-24T13:47:11,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031014 2024-11-24T13:47:11,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031014 2024-11-24T13:47:11,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,019 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031014) roll requested 2024-11-24T13:47:11,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031019 2024-11-24T13:47:11,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031019 2024-11-24T13:47:11,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,024 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031019) roll requested 2024-11-24T13:47:11,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031024 2024-11-24T13:47:11,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031024 2024-11-24T13:47:11,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031024) roll requested 2024-11-24T13:47:11,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031030 2024-11-24T13:47:11,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031030 2024-11-24T13:47:11,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031030) roll requested 2024-11-24T13:47:11,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031036 2024-11-24T13:47:11,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031030 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031036 2024-11-24T13:47:11,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031036) roll requested 2024-11-24T13:47:11,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031044 2024-11-24T13:47:11,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031044 2024-11-24T13:47:11,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031044) roll requested 2024-11-24T13:47:11,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031051 2024-11-24T13:47:11,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031051 2024-11-24T13:47:11,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,058 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031051) roll requested 2024-11-24T13:47:11,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031059 2024-11-24T13:47:11,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031059 2024-11-24T13:47:11,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,067 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031059) roll requested 2024-11-24T13:47:11,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031067 2024-11-24T13:47:11,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031067 2024-11-24T13:47:11,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031067) roll requested 2024-11-24T13:47:11,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031074 2024-11-24T13:47:11,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031067 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031074 2024-11-24T13:47:11,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,081 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031074) roll requested 2024-11-24T13:47:11,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031082 2024-11-24T13:47:11,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031074 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031082 2024-11-24T13:47:11,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031082) roll requested 2024-11-24T13:47:11,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031090 2024-11-24T13:47:11,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031090 2024-11-24T13:47:11,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031090) roll requested 2024-11-24T13:47:11,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031099 2024-11-24T13:47:11,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031099 2024-11-24T13:47:11,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031099) roll requested 2024-11-24T13:47:11,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031107 2024-11-24T13:47:11,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031107 2024-11-24T13:47:11,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031107) roll requested 2024-11-24T13:47:11,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031116 2024-11-24T13:47:11,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031116 2024-11-24T13:47:11,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,126 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031116) roll requested 2024-11-24T13:47:11,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031127 2024-11-24T13:47:11,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031127 2024-11-24T13:47:11,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031127) roll requested 2024-11-24T13:47:11,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031136 2024-11-24T13:47:11,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031136 2024-11-24T13:47:11,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031136) roll requested 2024-11-24T13:47:11,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031144 2024-11-24T13:47:11,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031144 2024-11-24T13:47:11,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031144) roll requested 2024-11-24T13:47:11,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031151 2024-11-24T13:47:11,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031144 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031151 2024-11-24T13:47:11,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031151) roll requested 2024-11-24T13:47:11,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031160 2024-11-24T13:47:11,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031160 2024-11-24T13:47:11,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031160) roll requested 2024-11-24T13:47:11,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031171 2024-11-24T13:47:11,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031171 2024-11-24T13:47:11,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031171) roll requested 2024-11-24T13:47:11,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031180 2024-11-24T13:47:11,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031180 2024-11-24T13:47:11,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,187 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031180) roll requested 2024-11-24T13:47:11,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031188 2024-11-24T13:47:11,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031188 2024-11-24T13:47:11,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031188) roll requested 2024-11-24T13:47:11,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031196 2024-11-24T13:47:11,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031196 2024-11-24T13:47:11,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031196) roll requested 2024-11-24T13:47:11,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031205 2024-11-24T13:47:11,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031205 2024-11-24T13:47:11,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031205) roll requested 2024-11-24T13:47:11,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031215 2024-11-24T13:47:11,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031215 2024-11-24T13:47:11,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031215) roll requested 2024-11-24T13:47:11,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031224 2024-11-24T13:47:11,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031224 2024-11-24T13:47:11,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031224) roll requested 2024-11-24T13:47:11,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031232 2024-11-24T13:47:11,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031232 2024-11-24T13:47:11,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031232) roll requested 2024-11-24T13:47:11,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031238 2024-11-24T13:47:11,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031238 2024-11-24T13:47:11,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,243 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031238) roll requested 2024-11-24T13:47:11,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031244 2024-11-24T13:47:11,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031244 2024-11-24T13:47:11,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031244) roll requested 2024-11-24T13:47:11,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031253 2024-11-24T13:47:11,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031253 2024-11-24T13:47:11,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031253) roll requested 2024-11-24T13:47:11,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031261 2024-11-24T13:47:11,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031261 2024-11-24T13:47:11,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031261) roll requested 2024-11-24T13:47:11,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031271 2024-11-24T13:47:11,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031271 2024-11-24T13:47:11,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031271) roll requested 2024-11-24T13:47:11,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031279 2024-11-24T13:47:11,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031279 2024-11-24T13:47:11,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031279) roll requested 2024-11-24T13:47:11,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031288 2024-11-24T13:47:11,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031288 2024-11-24T13:47:11,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,297 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031288) roll requested 2024-11-24T13:47:11,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031297 2024-11-24T13:47:11,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031297 2024-11-24T13:47:11,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,307 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031297) roll requested 2024-11-24T13:47:11,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031307 2024-11-24T13:47:11,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031307 2024-11-24T13:47:11,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031307) roll requested 2024-11-24T13:47:11,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031314 2024-11-24T13:47:11,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031314 2024-11-24T13:47:11,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031314) roll requested 2024-11-24T13:47:11,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031322 2024-11-24T13:47:11,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031322 2024-11-24T13:47:11,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,333 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031322) roll requested 2024-11-24T13:47:11,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031333 2024-11-24T13:47:11,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031322 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031333 2024-11-24T13:47:11,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031333) roll requested 2024-11-24T13:47:11,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031344 2024-11-24T13:47:11,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031344 2024-11-24T13:47:11,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031344) roll requested 2024-11-24T13:47:11,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031352 2024-11-24T13:47:11,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031352 2024-11-24T13:47:11,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031352) roll requested 2024-11-24T13:47:11,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031360 2024-11-24T13:47:11,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031360 2024-11-24T13:47:11,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031360) roll requested 2024-11-24T13:47:11,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031369 2024-11-24T13:47:11,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031369 2024-11-24T13:47:11,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031369) roll requested 2024-11-24T13:47:11,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031378 2024-11-24T13:47:11,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031378 2024-11-24T13:47:11,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031378) roll requested 2024-11-24T13:47:11,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031387 2024-11-24T13:47:11,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031387 2024-11-24T13:47:11,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031387) roll requested 2024-11-24T13:47:11,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031397 2024-11-24T13:47:11,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031397 2024-11-24T13:47:11,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031397) roll requested 2024-11-24T13:47:11,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031404 2024-11-24T13:47:11,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031404 2024-11-24T13:47:11,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031404) roll requested 2024-11-24T13:47:11,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031413 2024-11-24T13:47:11,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031413 2024-11-24T13:47:11,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031413) roll requested 2024-11-24T13:47:11,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031424 2024-11-24T13:47:11,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031424 2024-11-24T13:47:11,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031424) roll requested 2024-11-24T13:47:11,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031429 2024-11-24T13:47:11,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031429 2024-11-24T13:47:11,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031429) roll requested 2024-11-24T13:47:11,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031435 2024-11-24T13:47:11,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031435 2024-11-24T13:47:11,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031435) roll requested 2024-11-24T13:47:11,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031441 2024-11-24T13:47:11,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031441 2024-11-24T13:47:11,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031441) roll requested 2024-11-24T13:47:11,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031447 2024-11-24T13:47:11,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031447 2024-11-24T13:47:11,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031447) roll requested 2024-11-24T13:47:11,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031454 2024-11-24T13:47:11,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031454 2024-11-24T13:47:11,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031454) roll requested 2024-11-24T13:47:11,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031461 2024-11-24T13:47:11,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031461 2024-11-24T13:47:11,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031461) roll requested 2024-11-24T13:47:11,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031470 2024-11-24T13:47:11,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031470 2024-11-24T13:47:11,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031470) roll requested 2024-11-24T13:47:11,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031477 2024-11-24T13:47:11,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031470 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031477 2024-11-24T13:47:11,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031477) roll requested 2024-11-24T13:47:11,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031482 2024-11-24T13:47:11,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031482 2024-11-24T13:47:11,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031482) roll requested 2024-11-24T13:47:11,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031489 2024-11-24T13:47:11,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031482 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031489 2024-11-24T13:47:11,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031489) roll requested 2024-11-24T13:47:11,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031497 2024-11-24T13:47:11,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031497 2024-11-24T13:47:11,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,505 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031497) roll requested 2024-11-24T13:47:11,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031505 2024-11-24T13:47:11,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031505 2024-11-24T13:47:11,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031505) roll requested 2024-11-24T13:47:11,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031514 2024-11-24T13:47:11,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031514 2024-11-24T13:47:11,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031514) roll requested 2024-11-24T13:47:11,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031522 2024-11-24T13:47:11,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031522 2024-11-24T13:47:11,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031522) roll requested 2024-11-24T13:47:11,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031531 2024-11-24T13:47:11,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031531 2024-11-24T13:47:11,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031531) roll requested 2024-11-24T13:47:11,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031540 2024-11-24T13:47:11,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031540 2024-11-24T13:47:11,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031540) roll requested 2024-11-24T13:47:11,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031547 2024-11-24T13:47:11,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031540 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031547 2024-11-24T13:47:11,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031547) roll requested 2024-11-24T13:47:11,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031554 2024-11-24T13:47:11,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031554 2024-11-24T13:47:11,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,561 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031554) roll requested 2024-11-24T13:47:11,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031561 2024-11-24T13:47:11,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031554 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031561 2024-11-24T13:47:11,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031561) roll requested 2024-11-24T13:47:11,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031572 2024-11-24T13:47:11,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031572 2024-11-24T13:47:11,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,579 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031572) roll requested 2024-11-24T13:47:11,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031579 2024-11-24T13:47:11,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031579 2024-11-24T13:47:11,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,586 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031579) roll requested 2024-11-24T13:47:11,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031586 2024-11-24T13:47:11,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031579 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031586 2024-11-24T13:47:11,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031586) roll requested 2024-11-24T13:47:11,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031594 2024-11-24T13:47:11,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031586 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031594 2024-11-24T13:47:11,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031594) roll requested 2024-11-24T13:47:11,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031603 2024-11-24T13:47:11,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031603 2024-11-24T13:47:11,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031603) roll requested 2024-11-24T13:47:11,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031610 2024-11-24T13:47:11,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031610 2024-11-24T13:47:11,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031610) roll requested 2024-11-24T13:47:11,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031616 2024-11-24T13:47:11,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031610 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031616 2024-11-24T13:47:11,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031616) roll requested 2024-11-24T13:47:11,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031624 2024-11-24T13:47:11,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031624 2024-11-24T13:47:11,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031624) roll requested 2024-11-24T13:47:11,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031633 2024-11-24T13:47:11,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031633 2024-11-24T13:47:11,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031633) roll requested 2024-11-24T13:47:11,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031641 2024-11-24T13:47:11,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031633 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031641 2024-11-24T13:47:11,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031641) roll requested 2024-11-24T13:47:11,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031649 2024-11-24T13:47:11,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031649 2024-11-24T13:47:11,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031649) roll requested 2024-11-24T13:47:11,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031657 2024-11-24T13:47:11,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031657 2024-11-24T13:47:11,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031657) roll requested 2024-11-24T13:47:11,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031666 2024-11-24T13:47:11,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031666 2024-11-24T13:47:11,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031666) roll requested 2024-11-24T13:47:11,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031675 2024-11-24T13:47:11,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031666 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031675 2024-11-24T13:47:11,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031675) roll requested 2024-11-24T13:47:11,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031682 2024-11-24T13:47:11,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031675 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031682 2024-11-24T13:47:11,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031682) roll requested 2024-11-24T13:47:11,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031689 2024-11-24T13:47:11,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031689 2024-11-24T13:47:11,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031689) roll requested 2024-11-24T13:47:11,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031696 2024-11-24T13:47:11,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031696 2024-11-24T13:47:11,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031696) roll requested 2024-11-24T13:47:11,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031703 2024-11-24T13:47:11,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031703 2024-11-24T13:47:11,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031703) roll requested 2024-11-24T13:47:11,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031712 2024-11-24T13:47:11,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031703 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031712 2024-11-24T13:47:11,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031712) roll requested 2024-11-24T13:47:11,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031721 2024-11-24T13:47:11,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031721 2024-11-24T13:47:11,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031721) roll requested 2024-11-24T13:47:11,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031729 2024-11-24T13:47:11,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031729 2024-11-24T13:47:11,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031729) roll requested 2024-11-24T13:47:11,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031737 2024-11-24T13:47:11,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031737 2024-11-24T13:47:11,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,746 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031737) roll requested 2024-11-24T13:47:11,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031746 2024-11-24T13:47:11,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031746 2024-11-24T13:47:11,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,754 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031746) roll requested 2024-11-24T13:47:11,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031754 2024-11-24T13:47:11,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031754 2024-11-24T13:47:11,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031754) roll requested 2024-11-24T13:47:11,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031762 2024-11-24T13:47:11,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031754 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031762 2024-11-24T13:47:11,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031762) roll requested 2024-11-24T13:47:11,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031771 2024-11-24T13:47:11,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031762 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031771 2024-11-24T13:47:11,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031771) roll requested 2024-11-24T13:47:11,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031779 2024-11-24T13:47:11,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031779 2024-11-24T13:47:11,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031779) roll requested 2024-11-24T13:47:11,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031789 2024-11-24T13:47:11,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031789 2024-11-24T13:47:11,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,798 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031789) roll requested 2024-11-24T13:47:11,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031798 2024-11-24T13:47:11,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031798 2024-11-24T13:47:11,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031798) roll requested 2024-11-24T13:47:11,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031807 2024-11-24T13:47:11,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031798 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031807 2024-11-24T13:47:11,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031807) roll requested 2024-11-24T13:47:11,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031817 2024-11-24T13:47:11,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031807 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031817 2024-11-24T13:47:11,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,823 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031817) roll requested 2024-11-24T13:47:11,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031823 2024-11-24T13:47:11,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031823 2024-11-24T13:47:11,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031823) roll requested 2024-11-24T13:47:11,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031829 2024-11-24T13:47:11,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031823 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031829 2024-11-24T13:47:11,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031829) roll requested 2024-11-24T13:47:11,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031837 2024-11-24T13:47:11,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031837 2024-11-24T13:47:11,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031837) roll requested 2024-11-24T13:47:11,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031845 2024-11-24T13:47:11,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031845 2024-11-24T13:47:11,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031845) roll requested 2024-11-24T13:47:11,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031853 2024-11-24T13:47:11,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031853 2024-11-24T13:47:11,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031853) roll requested 2024-11-24T13:47:11,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031862 2024-11-24T13:47:11,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031862 2024-11-24T13:47:11,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,870 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031862) roll requested 2024-11-24T13:47:11,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031871 2024-11-24T13:47:11,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031871 2024-11-24T13:47:11,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,879 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031871) roll requested 2024-11-24T13:47:11,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031879 2024-11-24T13:47:11,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031879 2024-11-24T13:47:11,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031879) roll requested 2024-11-24T13:47:11,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031888 2024-11-24T13:47:11,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031879 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031888 2024-11-24T13:47:11,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031888) roll requested 2024-11-24T13:47:11,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031897 2024-11-24T13:47:11,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031897 2024-11-24T13:47:11,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031897) roll requested 2024-11-24T13:47:11,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031905 2024-11-24T13:47:11,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031897 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031905 2024-11-24T13:47:11,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,913 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031905) roll requested 2024-11-24T13:47:11,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031913 2024-11-24T13:47:11,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031913 2024-11-24T13:47:11,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,922 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031913) roll requested 2024-11-24T13:47:11,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031922 2024-11-24T13:47:11,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031922 2024-11-24T13:47:11,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,930 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031922) roll requested 2024-11-24T13:47:11,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031930 2024-11-24T13:47:11,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031930 2024-11-24T13:47:11,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031930) roll requested 2024-11-24T13:47:11,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031938 2024-11-24T13:47:11,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031930 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031938 2024-11-24T13:47:11,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031938) roll requested 2024-11-24T13:47:11,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031945 2024-11-24T13:47:11,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031945 2024-11-24T13:47:11,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031945) roll requested 2024-11-24T13:47:11,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031953 2024-11-24T13:47:11,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031953 2024-11-24T13:47:11,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031953) roll requested 2024-11-24T13:47:11,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031960 2024-11-24T13:47:11,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031960 2024-11-24T13:47:11,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,967 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031960) roll requested 2024-11-24T13:47:11,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031967 2024-11-24T13:47:11,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031967 2024-11-24T13:47:11,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031967) roll requested 2024-11-24T13:47:11,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031976 2024-11-24T13:47:11,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031976 2024-11-24T13:47:11,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031976) roll requested 2024-11-24T13:47:11,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031983 2024-11-24T13:47:11,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031983 2024-11-24T13:47:11,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,990 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031983) roll requested 2024-11-24T13:47:11,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031991 2024-11-24T13:47:11,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:11,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031991 2024-11-24T13:47:11,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:11,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:11,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031991) roll requested 2024-11-24T13:47:11,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456031997 2024-11-24T13:47:12,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031997 2024-11-24T13:47:12,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456031997) roll requested 2024-11-24T13:47:12,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032004 2024-11-24T13:47:12,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456031997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032004 2024-11-24T13:47:12,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032004) roll requested 2024-11-24T13:47:12,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032012 2024-11-24T13:47:12,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032012 2024-11-24T13:47:12,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032012) roll requested 2024-11-24T13:47:12,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032021 2024-11-24T13:47:12,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032021 2024-11-24T13:47:12,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032021) roll requested 2024-11-24T13:47:12,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032031 2024-11-24T13:47:12,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032031 2024-11-24T13:47:12,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032031) roll requested 2024-11-24T13:47:12,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032039 2024-11-24T13:47:12,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032031 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032039 2024-11-24T13:47:12,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,046 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032039) roll requested 2024-11-24T13:47:12,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032047 2024-11-24T13:47:12,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032047 2024-11-24T13:47:12,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032047) roll requested 2024-11-24T13:47:12,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032056 2024-11-24T13:47:12,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032056 2024-11-24T13:47:12,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032056) roll requested 2024-11-24T13:47:12,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032063 2024-11-24T13:47:12,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032063 2024-11-24T13:47:12,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032063) roll requested 2024-11-24T13:47:12,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032070 2024-11-24T13:47:12,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032063 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032070 2024-11-24T13:47:12,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032070) roll requested 2024-11-24T13:47:12,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032078 2024-11-24T13:47:12,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032078 2024-11-24T13:47:12,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032078) roll requested 2024-11-24T13:47:12,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032086 2024-11-24T13:47:12,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032078 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032086 2024-11-24T13:47:12,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032086) roll requested 2024-11-24T13:47:12,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032092 2024-11-24T13:47:12,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032092 2024-11-24T13:47:12,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032092) roll requested 2024-11-24T13:47:12,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032100 2024-11-24T13:47:12,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032100 2024-11-24T13:47:12,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032100) roll requested 2024-11-24T13:47:12,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032109 2024-11-24T13:47:12,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032109 2024-11-24T13:47:12,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032100 is not closed yet, will try archiving it next time 2024-11-24T13:47:12,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032109) roll requested 2024-11-24T13:47:12,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032118 2024-11-24T13:47:12,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032118 2024-11-24T13:47:12,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032118) roll requested 2024-11-24T13:47:12,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032166 2024-11-24T13:47:12,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032118 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032166 2024-11-24T13:47:12,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032166) roll requested 2024-11-24T13:47:12,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032173 2024-11-24T13:47:12,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032166 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032173 2024-11-24T13:47:12,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032173) roll requested 2024-11-24T13:47:12,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032183 2024-11-24T13:47:12,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032183 2024-11-24T13:47:12,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,193 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032183) roll requested 2024-11-24T13:47:12,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032193 2024-11-24T13:47:12,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032193 2024-11-24T13:47:12,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032193) roll requested 2024-11-24T13:47:12,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032202 2024-11-24T13:47:12,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032202 2024-11-24T13:47:12,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032202) roll requested 2024-11-24T13:47:12,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032210 2024-11-24T13:47:12,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032210 2024-11-24T13:47:12,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032210) roll requested 2024-11-24T13:47:12,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032218 2024-11-24T13:47:12,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032218 2024-11-24T13:47:12,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032218) roll requested 2024-11-24T13:47:12,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032223 2024-11-24T13:47:12,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032218 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032223 2024-11-24T13:47:12,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032223) roll requested 2024-11-24T13:47:12,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032228 2024-11-24T13:47:12,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032228 2024-11-24T13:47:12,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032228) roll requested 2024-11-24T13:47:12,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032233 2024-11-24T13:47:12,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032233 2024-11-24T13:47:12,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032233) roll requested 2024-11-24T13:47:12,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032239 2024-11-24T13:47:12,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032239 2024-11-24T13:47:12,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032239) roll requested 2024-11-24T13:47:12,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032245 2024-11-24T13:47:12,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032245 2024-11-24T13:47:12,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032245) roll requested 2024-11-24T13:47:12,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032251 2024-11-24T13:47:12,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032251 2024-11-24T13:47:12,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032251) roll requested 2024-11-24T13:47:12,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032257 2024-11-24T13:47:12,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032257 2024-11-24T13:47:12,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,265 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032257) roll requested 2024-11-24T13:47:12,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032265 2024-11-24T13:47:12,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032265 2024-11-24T13:47:12,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032265) roll requested 2024-11-24T13:47:12,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032274 2024-11-24T13:47:12,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032274 2024-11-24T13:47:12,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,282 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032274) roll requested 2024-11-24T13:47:12,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032282 2024-11-24T13:47:12,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032282 2024-11-24T13:47:12,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,290 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032282) roll requested 2024-11-24T13:47:12,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032291 2024-11-24T13:47:12,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032291 2024-11-24T13:47:12,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032291) roll requested 2024-11-24T13:47:12,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032300 2024-11-24T13:47:12,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032291 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032300 2024-11-24T13:47:12,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,309 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032300) roll requested 2024-11-24T13:47:12,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032309 2024-11-24T13:47:12,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032309 2024-11-24T13:47:12,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032309) roll requested 2024-11-24T13:47:12,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032316 2024-11-24T13:47:12,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032316 2024-11-24T13:47:12,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032316) roll requested 2024-11-24T13:47:12,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032323 2024-11-24T13:47:12,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032316 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032323 2024-11-24T13:47:12,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032323) roll requested 2024-11-24T13:47:12,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032328 2024-11-24T13:47:12,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032328 2024-11-24T13:47:12,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,333 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032328) roll requested 2024-11-24T13:47:12,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032333 2024-11-24T13:47:12,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032333 2024-11-24T13:47:12,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032333) roll requested 2024-11-24T13:47:12,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032339 2024-11-24T13:47:12,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032339 2024-11-24T13:47:12,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032339) roll requested 2024-11-24T13:47:12,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032346 2024-11-24T13:47:12,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032346 2024-11-24T13:47:12,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032346) roll requested 2024-11-24T13:47:12,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032355 2024-11-24T13:47:12,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032355 2024-11-24T13:47:12,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032355) roll requested 2024-11-24T13:47:12,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032363 2024-11-24T13:47:12,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032363 2024-11-24T13:47:12,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032363) roll requested 2024-11-24T13:47:12,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032370 2024-11-24T13:47:12,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032370 2024-11-24T13:47:12,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032370) roll requested 2024-11-24T13:47:12,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032379 2024-11-24T13:47:12,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032379 2024-11-24T13:47:12,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,385 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032379) roll requested 2024-11-24T13:47:12,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032385 2024-11-24T13:47:12,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032385 2024-11-24T13:47:12,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,391 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032385) roll requested 2024-11-24T13:47:12,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032392 2024-11-24T13:47:12,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032392 2024-11-24T13:47:12,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032392) roll requested 2024-11-24T13:47:12,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032399 2024-11-24T13:47:12,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032399 2024-11-24T13:47:12,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032399) roll requested 2024-11-24T13:47:12,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032408 2024-11-24T13:47:12,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032408 2024-11-24T13:47:12,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,417 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032408) roll requested 2024-11-24T13:47:12,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032417 2024-11-24T13:47:12,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032408 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032417 2024-11-24T13:47:12,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,425 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032417) roll requested 2024-11-24T13:47:12,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032425 2024-11-24T13:47:12,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032417 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032425 2024-11-24T13:47:12,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032425) roll requested 2024-11-24T13:47:12,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032433 2024-11-24T13:47:12,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032433 2024-11-24T13:47:12,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,442 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032433) roll requested 2024-11-24T13:47:12,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032442 2024-11-24T13:47:12,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032442 2024-11-24T13:47:12,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032442) roll requested 2024-11-24T13:47:12,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032451 2024-11-24T13:47:12,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032451 2024-11-24T13:47:12,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032451) roll requested 2024-11-24T13:47:12,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032459 2024-11-24T13:47:12,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032459 2024-11-24T13:47:12,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032459) roll requested 2024-11-24T13:47:12,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032467 2024-11-24T13:47:12,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032467 2024-11-24T13:47:12,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032467) roll requested 2024-11-24T13:47:12,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032475 2024-11-24T13:47:12,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032475 2024-11-24T13:47:12,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032475) roll requested 2024-11-24T13:47:12,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032483 2024-11-24T13:47:12,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032483 2024-11-24T13:47:12,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032483) roll requested 2024-11-24T13:47:12,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032490 2024-11-24T13:47:12,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032490 2024-11-24T13:47:12,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,498 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032490) roll requested 2024-11-24T13:47:12,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032498 2024-11-24T13:47:12,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032498 2024-11-24T13:47:12,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032498) roll requested 2024-11-24T13:47:12,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032506 2024-11-24T13:47:12,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032498 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032506 2024-11-24T13:47:12,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032506) roll requested 2024-11-24T13:47:12,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032515 2024-11-24T13:47:12,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032515 2024-11-24T13:47:12,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,523 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032515) roll requested 2024-11-24T13:47:12,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032523 2024-11-24T13:47:12,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032523 2024-11-24T13:47:12,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,529 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032523) roll requested 2024-11-24T13:47:12,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032530 2024-11-24T13:47:12,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032523 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032530 2024-11-24T13:47:12,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032530) roll requested 2024-11-24T13:47:12,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032537 2024-11-24T13:47:12,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032537 2024-11-24T13:47:12,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032537) roll requested 2024-11-24T13:47:12,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032544 2024-11-24T13:47:12,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032544 2024-11-24T13:47:12,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032544) roll requested 2024-11-24T13:47:12,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032549 2024-11-24T13:47:12,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032549 2024-11-24T13:47:12,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032549) roll requested 2024-11-24T13:47:12,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032556 2024-11-24T13:47:12,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032556 2024-11-24T13:47:12,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,563 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032556) roll requested 2024-11-24T13:47:12,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032563 2024-11-24T13:47:12,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032563 2024-11-24T13:47:12,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032563) roll requested 2024-11-24T13:47:12,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032571 2024-11-24T13:47:12,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032571 2024-11-24T13:47:12,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032571) roll requested 2024-11-24T13:47:12,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032578 2024-11-24T13:47:12,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032578 2024-11-24T13:47:12,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032578) roll requested 2024-11-24T13:47:12,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032585 2024-11-24T13:47:12,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032585 2024-11-24T13:47:12,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032585) roll requested 2024-11-24T13:47:12,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032592 2024-11-24T13:47:12,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032592 2024-11-24T13:47:12,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032592) roll requested 2024-11-24T13:47:12,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032605 2024-11-24T13:47:12,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032605 2024-11-24T13:47:12,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032605) roll requested 2024-11-24T13:47:12,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032618 2024-11-24T13:47:12,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032618 2024-11-24T13:47:12,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032618) roll requested 2024-11-24T13:47:12,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032631 2024-11-24T13:47:12,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032631 2024-11-24T13:47:12,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032631) roll requested 2024-11-24T13:47:12,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032640 2024-11-24T13:47:12,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032640 2024-11-24T13:47:12,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032640) roll requested 2024-11-24T13:47:12,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032645 2024-11-24T13:47:12,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032645 2024-11-24T13:47:12,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032645) roll requested 2024-11-24T13:47:12,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032651 2024-11-24T13:47:12,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032651 2024-11-24T13:47:12,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032651) roll requested 2024-11-24T13:47:12,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032656 2024-11-24T13:47:12,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032656 2024-11-24T13:47:12,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,661 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032656) roll requested 2024-11-24T13:47:12,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032662 2024-11-24T13:47:12,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032662 2024-11-24T13:47:12,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,666 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032662) roll requested 2024-11-24T13:47:12,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032666 2024-11-24T13:47:12,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032666 2024-11-24T13:47:12,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,673 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032666) roll requested 2024-11-24T13:47:12,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032673 2024-11-24T13:47:12,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032666 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032673 2024-11-24T13:47:12,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032673) roll requested 2024-11-24T13:47:12,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032679 2024-11-24T13:47:12,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032679 2024-11-24T13:47:12,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,687 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032679) roll requested 2024-11-24T13:47:12,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032687 2024-11-24T13:47:12,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032687 2024-11-24T13:47:12,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032687) roll requested 2024-11-24T13:47:12,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032695 2024-11-24T13:47:12,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032687 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032695 2024-11-24T13:47:12,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032695) roll requested 2024-11-24T13:47:12,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032702 2024-11-24T13:47:12,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032702 2024-11-24T13:47:12,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032702) roll requested 2024-11-24T13:47:12,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032708 2024-11-24T13:47:12,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032708 2024-11-24T13:47:12,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032708) roll requested 2024-11-24T13:47:12,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032714 2024-11-24T13:47:12,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032714 2024-11-24T13:47:12,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032714) roll requested 2024-11-24T13:47:12,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032721 2024-11-24T13:47:12,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032721 2024-11-24T13:47:12,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,730 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032721) roll requested 2024-11-24T13:47:12,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032730 2024-11-24T13:47:12,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032730 2024-11-24T13:47:12,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,738 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032730) roll requested 2024-11-24T13:47:12,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032738 2024-11-24T13:47:12,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032738 2024-11-24T13:47:12,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032738) roll requested 2024-11-24T13:47:12,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032744 2024-11-24T13:47:12,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032738 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032744 2024-11-24T13:47:12,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032744) roll requested 2024-11-24T13:47:12,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032749 2024-11-24T13:47:12,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032749 2024-11-24T13:47:12,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032749) roll requested 2024-11-24T13:47:12,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032756 2024-11-24T13:47:12,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032756 2024-11-24T13:47:12,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032756) roll requested 2024-11-24T13:47:12,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032764 2024-11-24T13:47:12,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032764 2024-11-24T13:47:12,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032764) roll requested 2024-11-24T13:47:12,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032773 2024-11-24T13:47:12,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032773 2024-11-24T13:47:12,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032773) roll requested 2024-11-24T13:47:12,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032782 2024-11-24T13:47:12,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032782 2024-11-24T13:47:12,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032782) roll requested 2024-11-24T13:47:12,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032791 2024-11-24T13:47:12,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032791 2024-11-24T13:47:12,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032791) roll requested 2024-11-24T13:47:12,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032801 2024-11-24T13:47:12,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032801 2024-11-24T13:47:12,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032801) roll requested 2024-11-24T13:47:12,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032808 2024-11-24T13:47:12,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032801 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032808 2024-11-24T13:47:12,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032808) roll requested 2024-11-24T13:47:12,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032817 2024-11-24T13:47:12,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032817 2024-11-24T13:47:12,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032817) roll requested 2024-11-24T13:47:12,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032826 2024-11-24T13:47:12,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032826 2024-11-24T13:47:12,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032826) roll requested 2024-11-24T13:47:12,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032833 2024-11-24T13:47:12,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032833 2024-11-24T13:47:12,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032833) roll requested 2024-11-24T13:47:12,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032841 2024-11-24T13:47:12,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032841 2024-11-24T13:47:12,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032841) roll requested 2024-11-24T13:47:12,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032848 2024-11-24T13:47:12,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032848 2024-11-24T13:47:12,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032848) roll requested 2024-11-24T13:47:12,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032856 2024-11-24T13:47:12,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032856 2024-11-24T13:47:12,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032856) roll requested 2024-11-24T13:47:12,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032865 2024-11-24T13:47:12,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032865 2024-11-24T13:47:12,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,874 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032865) roll requested 2024-11-24T13:47:12,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032874 2024-11-24T13:47:12,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032874 2024-11-24T13:47:12,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032874) roll requested 2024-11-24T13:47:12,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032881 2024-11-24T13:47:12,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032881 2024-11-24T13:47:12,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032881) roll requested 2024-11-24T13:47:12,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032888 2024-11-24T13:47:12,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032888 2024-11-24T13:47:12,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032888) roll requested 2024-11-24T13:47:12,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032895 2024-11-24T13:47:12,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032895 2024-11-24T13:47:12,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032895) roll requested 2024-11-24T13:47:12,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032902 2024-11-24T13:47:12,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032902 2024-11-24T13:47:12,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032902) roll requested 2024-11-24T13:47:12,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032910 2024-11-24T13:47:12,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032910 2024-11-24T13:47:12,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032910) roll requested 2024-11-24T13:47:12,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032921 2024-11-24T13:47:12,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032910 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032921 2024-11-24T13:47:12,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032921) roll requested 2024-11-24T13:47:12,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032931 2024-11-24T13:47:12,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032931 2024-11-24T13:47:12,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032931) roll requested 2024-11-24T13:47:12,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032937 2024-11-24T13:47:12,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032937 2024-11-24T13:47:12,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032937) roll requested 2024-11-24T13:47:12,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032943 2024-11-24T13:47:12,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032943 2024-11-24T13:47:12,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032943) roll requested 2024-11-24T13:47:12,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032951 2024-11-24T13:47:12,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032943 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032951 2024-11-24T13:47:12,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032951) roll requested 2024-11-24T13:47:12,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032960 2024-11-24T13:47:12,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032951 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032960 2024-11-24T13:47:12,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032960) roll requested 2024-11-24T13:47:12,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032969 2024-11-24T13:47:12,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032969 2024-11-24T13:47:12,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032969) roll requested 2024-11-24T13:47:12,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032977 2024-11-24T13:47:12,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032977 2024-11-24T13:47:12,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032977) roll requested 2024-11-24T13:47:12,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032983 2024-11-24T13:47:12,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032983 2024-11-24T13:47:12,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032983) roll requested 2024-11-24T13:47:12,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032988 2024-11-24T13:47:12,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:12,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032988 2024-11-24T13:47:12,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:12,994 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:12,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032988) roll requested 2024-11-24T13:47:12,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456032994 2024-11-24T13:47:13,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032994 2024-11-24T13:47:13,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456032994) roll requested 2024-11-24T13:47:13,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033002 2024-11-24T13:47:13,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456032994 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033002 2024-11-24T13:47:13,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033002) roll requested 2024-11-24T13:47:13,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033009 2024-11-24T13:47:13,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033009 2024-11-24T13:47:13,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033009) roll requested 2024-11-24T13:47:13,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033017 2024-11-24T13:47:13,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033017 2024-11-24T13:47:13,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033017) roll requested 2024-11-24T13:47:13,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033025 2024-11-24T13:47:13,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033025 2024-11-24T13:47:13,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033025) roll requested 2024-11-24T13:47:13,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033032 2024-11-24T13:47:13,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033032 2024-11-24T13:47:13,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033032) roll requested 2024-11-24T13:47:13,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033039 2024-11-24T13:47:13,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033039 2024-11-24T13:47:13,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,046 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033039) roll requested 2024-11-24T13:47:13,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033046 2024-11-24T13:47:13,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033046 2024-11-24T13:47:13,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,054 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033046) roll requested 2024-11-24T13:47:13,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033054 2024-11-24T13:47:13,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033046 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033054 2024-11-24T13:47:13,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033054) roll requested 2024-11-24T13:47:13,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033060 2024-11-24T13:47:13,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033060 2024-11-24T13:47:13,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033060) roll requested 2024-11-24T13:47:13,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033065 2024-11-24T13:47:13,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033060 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033065 2024-11-24T13:47:13,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033065) roll requested 2024-11-24T13:47:13,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033070 2024-11-24T13:47:13,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033070 2024-11-24T13:47:13,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033070) roll requested 2024-11-24T13:47:13,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033075 2024-11-24T13:47:13,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033075 2024-11-24T13:47:13,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033075) roll requested 2024-11-24T13:47:13,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033082 2024-11-24T13:47:13,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033082 2024-11-24T13:47:13,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033082) roll requested 2024-11-24T13:47:13,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033089 2024-11-24T13:47:13,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033089 2024-11-24T13:47:13,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033089) roll requested 2024-11-24T13:47:13,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033097 2024-11-24T13:47:13,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033097 2024-11-24T13:47:13,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033097) roll requested 2024-11-24T13:47:13,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033103 2024-11-24T13:47:13,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033103 2024-11-24T13:47:13,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,110 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033103) roll requested 2024-11-24T13:47:13,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033110 2024-11-24T13:47:13,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033110 2024-11-24T13:47:13,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033110) roll requested 2024-11-24T13:47:13,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033119 2024-11-24T13:47:13,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033110 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033119 2024-11-24T13:47:13,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,126 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033119) roll requested 2024-11-24T13:47:13,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033126 2024-11-24T13:47:13,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033126 2024-11-24T13:47:13,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,133 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033126) roll requested 2024-11-24T13:47:13,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033133 2024-11-24T13:47:13,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033126 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033133 2024-11-24T13:47:13,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033133) roll requested 2024-11-24T13:47:13,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033141 2024-11-24T13:47:13,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033141 2024-11-24T13:47:13,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033141) roll requested 2024-11-24T13:47:13,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033148 2024-11-24T13:47:13,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033148 2024-11-24T13:47:13,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,155 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033148) roll requested 2024-11-24T13:47:13,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033155 2024-11-24T13:47:13,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033155 2024-11-24T13:47:13,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,162 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033155) roll requested 2024-11-24T13:47:13,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033162 2024-11-24T13:47:13,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033155 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033162 2024-11-24T13:47:13,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033162) roll requested 2024-11-24T13:47:13,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033168 2024-11-24T13:47:13,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033162 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033168 2024-11-24T13:47:13,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,174 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033168) roll requested 2024-11-24T13:47:13,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033174 2024-11-24T13:47:13,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033174 2024-11-24T13:47:13,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,181 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033174) roll requested 2024-11-24T13:47:13,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033181 2024-11-24T13:47:13,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033174 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033181 2024-11-24T13:47:13,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033181) roll requested 2024-11-24T13:47:13,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033189 2024-11-24T13:47:13,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033189 2024-11-24T13:47:13,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033189) roll requested 2024-11-24T13:47:13,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033197 2024-11-24T13:47:13,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033197 2024-11-24T13:47:13,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033197) roll requested 2024-11-24T13:47:13,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033205 2024-11-24T13:47:13,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033205 2024-11-24T13:47:13,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033205) roll requested 2024-11-24T13:47:13,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033214 2024-11-24T13:47:13,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033214 2024-11-24T13:47:13,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033214) roll requested 2024-11-24T13:47:13,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033221 2024-11-24T13:47:13,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033221 2024-11-24T13:47:13,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033221) roll requested 2024-11-24T13:47:13,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033229 2024-11-24T13:47:13,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033229 2024-11-24T13:47:13,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033229) roll requested 2024-11-24T13:47:13,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033238 2024-11-24T13:47:13,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033238 2024-11-24T13:47:13,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033238) roll requested 2024-11-24T13:47:13,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033247 2024-11-24T13:47:13,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033247 2024-11-24T13:47:13,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033247) roll requested 2024-11-24T13:47:13,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033255 2024-11-24T13:47:13,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033255 2024-11-24T13:47:13,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,260 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033255) roll requested 2024-11-24T13:47:13,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033260 2024-11-24T13:47:13,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033260 2024-11-24T13:47:13,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033260) roll requested 2024-11-24T13:47:13,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033267 2024-11-24T13:47:13,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033267 2024-11-24T13:47:13,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033267) roll requested 2024-11-24T13:47:13,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033273 2024-11-24T13:47:13,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033273 2024-11-24T13:47:13,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,282 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033273) roll requested 2024-11-24T13:47:13,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033283 2024-11-24T13:47:13,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033283 2024-11-24T13:47:13,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033283) roll requested 2024-11-24T13:47:13,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033288 2024-11-24T13:47:13,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033283 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033288 2024-11-24T13:47:13,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033288) roll requested 2024-11-24T13:47:13,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033294 2024-11-24T13:47:13,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033294 2024-11-24T13:47:13,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033294) roll requested 2024-11-24T13:47:13,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033299 2024-11-24T13:47:13,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033299 2024-11-24T13:47:13,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033299) roll requested 2024-11-24T13:47:13,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033305 2024-11-24T13:47:13,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033305 2024-11-24T13:47:13,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033305) roll requested 2024-11-24T13:47:13,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033310 2024-11-24T13:47:13,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033310 2024-11-24T13:47:13,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033310) roll requested 2024-11-24T13:47:13,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033316 2024-11-24T13:47:13,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033316 2024-11-24T13:47:13,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033316) roll requested 2024-11-24T13:47:13,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033321 2024-11-24T13:47:13,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033316 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033321 2024-11-24T13:47:13,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033321) roll requested 2024-11-24T13:47:13,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033327 2024-11-24T13:47:13,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033327 2024-11-24T13:47:13,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033327) roll requested 2024-11-24T13:47:13,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033335 2024-11-24T13:47:13,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033335 2024-11-24T13:47:13,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033335) roll requested 2024-11-24T13:47:13,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033340 2024-11-24T13:47:13,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033340 2024-11-24T13:47:13,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033340) roll requested 2024-11-24T13:47:13,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033345 2024-11-24T13:47:13,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033345 2024-11-24T13:47:13,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033345) roll requested 2024-11-24T13:47:13,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033350 2024-11-24T13:47:13,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033350 2024-11-24T13:47:13,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033350) roll requested 2024-11-24T13:47:13,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033355 2024-11-24T13:47:13,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033355 2024-11-24T13:47:13,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033355) roll requested 2024-11-24T13:47:13,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033362 2024-11-24T13:47:13,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033362 2024-11-24T13:47:13,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,367 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033362) roll requested 2024-11-24T13:47:13,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033368 2024-11-24T13:47:13,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033368 2024-11-24T13:47:13,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033368) roll requested 2024-11-24T13:47:13,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033374 2024-11-24T13:47:13,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033374 2024-11-24T13:47:13,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,381 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033374) roll requested 2024-11-24T13:47:13,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033381 2024-11-24T13:47:13,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033381 2024-11-24T13:47:13,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,388 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033381) roll requested 2024-11-24T13:47:13,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033388 2024-11-24T13:47:13,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033381 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033388 2024-11-24T13:47:13,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033388) roll requested 2024-11-24T13:47:13,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033397 2024-11-24T13:47:13,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033397 2024-11-24T13:47:13,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033397) roll requested 2024-11-24T13:47:13,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033405 2024-11-24T13:47:13,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033405 2024-11-24T13:47:13,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033405) roll requested 2024-11-24T13:47:13,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033413 2024-11-24T13:47:13,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033413 2024-11-24T13:47:13,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033413) roll requested 2024-11-24T13:47:13,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033421 2024-11-24T13:47:13,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033421 2024-11-24T13:47:13,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033421) roll requested 2024-11-24T13:47:13,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033429 2024-11-24T13:47:13,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033429 2024-11-24T13:47:13,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033429) roll requested 2024-11-24T13:47:13,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033436 2024-11-24T13:47:13,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033436 2024-11-24T13:47:13,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,444 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033436) roll requested 2024-11-24T13:47:13,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033444 2024-11-24T13:47:13,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033444 2024-11-24T13:47:13,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033444) roll requested 2024-11-24T13:47:13,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033453 2024-11-24T13:47:13,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033453 2024-11-24T13:47:13,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033453) roll requested 2024-11-24T13:47:13,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033459 2024-11-24T13:47:13,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033459 2024-11-24T13:47:13,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033459) roll requested 2024-11-24T13:47:13,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033465 2024-11-24T13:47:13,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033465 2024-11-24T13:47:13,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033465) roll requested 2024-11-24T13:47:13,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033472 2024-11-24T13:47:13,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033472 2024-11-24T13:47:13,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033472) roll requested 2024-11-24T13:47:13,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033480 2024-11-24T13:47:13,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033480 2024-11-24T13:47:13,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033480) roll requested 2024-11-24T13:47:13,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033491 2024-11-24T13:47:13,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033491 2024-11-24T13:47:13,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033491) roll requested 2024-11-24T13:47:13,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033496 2024-11-24T13:47:13,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033496 2024-11-24T13:47:13,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033496) roll requested 2024-11-24T13:47:13,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033502 2024-11-24T13:47:13,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033502 2024-11-24T13:47:13,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033502) roll requested 2024-11-24T13:47:13,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033507 2024-11-24T13:47:13,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033507 2024-11-24T13:47:13,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033507) roll requested 2024-11-24T13:47:13,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033514 2024-11-24T13:47:13,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033514 2024-11-24T13:47:13,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033514) roll requested 2024-11-24T13:47:13,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033520 2024-11-24T13:47:13,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033520 2024-11-24T13:47:13,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033520) roll requested 2024-11-24T13:47:13,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033528 2024-11-24T13:47:13,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033528 2024-11-24T13:47:13,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033528) roll requested 2024-11-24T13:47:13,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033535 2024-11-24T13:47:13,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033535 2024-11-24T13:47:13,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033535) roll requested 2024-11-24T13:47:13,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033544 2024-11-24T13:47:13,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033544 2024-11-24T13:47:13,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033544) roll requested 2024-11-24T13:47:13,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033553 2024-11-24T13:47:13,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033553 2024-11-24T13:47:13,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033553) roll requested 2024-11-24T13:47:13,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033560 2024-11-24T13:47:13,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033560 2024-11-24T13:47:13,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,567 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033560) roll requested 2024-11-24T13:47:13,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033567 2024-11-24T13:47:13,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033567 2024-11-24T13:47:13,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033567) roll requested 2024-11-24T13:47:13,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033576 2024-11-24T13:47:13,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033576 2024-11-24T13:47:13,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033576) roll requested 2024-11-24T13:47:13,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033581 2024-11-24T13:47:13,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033581 2024-11-24T13:47:13,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033581) roll requested 2024-11-24T13:47:13,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033587 2024-11-24T13:47:13,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033587 2024-11-24T13:47:13,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033587) roll requested 2024-11-24T13:47:13,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033594 2024-11-24T13:47:13,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033594 2024-11-24T13:47:13,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033594) roll requested 2024-11-24T13:47:13,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033602 2024-11-24T13:47:13,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033602 2024-11-24T13:47:13,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033602) roll requested 2024-11-24T13:47:13,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033611 2024-11-24T13:47:13,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033611 2024-11-24T13:47:13,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033611) roll requested 2024-11-24T13:47:13,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033616 2024-11-24T13:47:13,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033611 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033616 2024-11-24T13:47:13,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033616) roll requested 2024-11-24T13:47:13,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033621 2024-11-24T13:47:13,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033621 2024-11-24T13:47:13,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033621) roll requested 2024-11-24T13:47:13,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033628 2024-11-24T13:47:13,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033628 2024-11-24T13:47:13,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,635 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033628) roll requested 2024-11-24T13:47:13,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033636 2024-11-24T13:47:13,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033636 2024-11-24T13:47:13,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033636) roll requested 2024-11-24T13:47:13,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033644 2024-11-24T13:47:13,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033644 2024-11-24T13:47:13,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033644) roll requested 2024-11-24T13:47:13,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033654 2024-11-24T13:47:13,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033654 2024-11-24T13:47:13,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033654) roll requested 2024-11-24T13:47:13,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033662 2024-11-24T13:47:13,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033662 2024-11-24T13:47:13,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033662) roll requested 2024-11-24T13:47:13,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033670 2024-11-24T13:47:13,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033670 2024-11-24T13:47:13,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,678 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033670) roll requested 2024-11-24T13:47:13,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033678 2024-11-24T13:47:13,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033678 2024-11-24T13:47:13,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,687 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033678) roll requested 2024-11-24T13:47:13,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033687 2024-11-24T13:47:13,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033687 2024-11-24T13:47:13,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033687) roll requested 2024-11-24T13:47:13,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033695 2024-11-24T13:47:13,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033687 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033695 2024-11-24T13:47:13,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033695) roll requested 2024-11-24T13:47:13,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033705 2024-11-24T13:47:13,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033705 2024-11-24T13:47:13,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033705) roll requested 2024-11-24T13:47:13,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033713 2024-11-24T13:47:13,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033713 2024-11-24T13:47:13,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033713) roll requested 2024-11-24T13:47:13,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033722 2024-11-24T13:47:13,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033722 2024-11-24T13:47:13,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033722) roll requested 2024-11-24T13:47:13,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033729 2024-11-24T13:47:13,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033722 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033729 2024-11-24T13:47:13,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033729) roll requested 2024-11-24T13:47:13,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033737 2024-11-24T13:47:13,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033737 2024-11-24T13:47:13,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033737) roll requested 2024-11-24T13:47:13,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033744 2024-11-24T13:47:13,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033744 2024-11-24T13:47:13,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033744) roll requested 2024-11-24T13:47:13,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033753 2024-11-24T13:47:13,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033753 2024-11-24T13:47:13,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033753) roll requested 2024-11-24T13:47:13,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033763 2024-11-24T13:47:13,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033763 2024-11-24T13:47:13,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033763) roll requested 2024-11-24T13:47:13,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033773 2024-11-24T13:47:13,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033773 2024-11-24T13:47:13,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033773) roll requested 2024-11-24T13:47:13,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033782 2024-11-24T13:47:13,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033782 2024-11-24T13:47:13,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033782) roll requested 2024-11-24T13:47:13,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033791 2024-11-24T13:47:13,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033791 2024-11-24T13:47:13,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033791) roll requested 2024-11-24T13:47:13,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033801 2024-11-24T13:47:13,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033801 2024-11-24T13:47:13,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033801) roll requested 2024-11-24T13:47:13,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033812 2024-11-24T13:47:13,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033801 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033812 2024-11-24T13:47:13,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033812) roll requested 2024-11-24T13:47:13,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033821 2024-11-24T13:47:13,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033821 2024-11-24T13:47:13,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033821) roll requested 2024-11-24T13:47:13,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033826 2024-11-24T13:47:13,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033826 2024-11-24T13:47:13,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033826) roll requested 2024-11-24T13:47:13,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033832 2024-11-24T13:47:13,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033832 2024-11-24T13:47:13,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033832) roll requested 2024-11-24T13:47:13,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033840 2024-11-24T13:47:13,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033840 2024-11-24T13:47:13,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033840) roll requested 2024-11-24T13:47:13,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033848 2024-11-24T13:47:13,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033848 2024-11-24T13:47:13,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033848) roll requested 2024-11-24T13:47:13,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033856 2024-11-24T13:47:13,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033856 2024-11-24T13:47:13,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033856) roll requested 2024-11-24T13:47:13,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033865 2024-11-24T13:47:13,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033865 2024-11-24T13:47:13,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033865) roll requested 2024-11-24T13:47:13,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033874 2024-11-24T13:47:13,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033874 2024-11-24T13:47:13,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,886 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033874) roll requested 2024-11-24T13:47:13,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033887 2024-11-24T13:47:13,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033887 2024-11-24T13:47:13,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033887) roll requested 2024-11-24T13:47:13,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033893 2024-11-24T13:47:13,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033893 2024-11-24T13:47:13,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033893) roll requested 2024-11-24T13:47:13,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033902 2024-11-24T13:47:13,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033902 2024-11-24T13:47:13,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033902) roll requested 2024-11-24T13:47:13,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033911 2024-11-24T13:47:13,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033911 2024-11-24T13:47:13,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033911) roll requested 2024-11-24T13:47:13,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033919 2024-11-24T13:47:13,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033919 2024-11-24T13:47:13,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033919) roll requested 2024-11-24T13:47:13,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033924 2024-11-24T13:47:13,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033924 2024-11-24T13:47:13,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,930 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033924) roll requested 2024-11-24T13:47:13,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033931 2024-11-24T13:47:13,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033931 2024-11-24T13:47:13,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033931) roll requested 2024-11-24T13:47:13,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033939 2024-11-24T13:47:13,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033939 2024-11-24T13:47:13,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033939) roll requested 2024-11-24T13:47:13,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033944 2024-11-24T13:47:13,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033944 2024-11-24T13:47:13,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,950 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033944) roll requested 2024-11-24T13:47:13,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033950 2024-11-24T13:47:13,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033950 2024-11-24T13:47:13,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033950) roll requested 2024-11-24T13:47:13,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033955 2024-11-24T13:47:13,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033955 2024-11-24T13:47:13,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033955) roll requested 2024-11-24T13:47:13,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033961 2024-11-24T13:47:13,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033955 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033961 2024-11-24T13:47:13,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033961) roll requested 2024-11-24T13:47:13,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033967 2024-11-24T13:47:13,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033967 2024-11-24T13:47:13,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,974 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033967) roll requested 2024-11-24T13:47:13,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033975 2024-11-24T13:47:13,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033975 2024-11-24T13:47:13,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033975) roll requested 2024-11-24T13:47:13,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033980 2024-11-24T13:47:13,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033975 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033980 2024-11-24T13:47:13,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033980) roll requested 2024-11-24T13:47:13,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033985 2024-11-24T13:47:13,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033985 2024-11-24T13:47:13,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033985) roll requested 2024-11-24T13:47:13,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033991 2024-11-24T13:47:13,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:13,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033991 2024-11-24T13:47:13,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:13,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:13,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033991) roll requested 2024-11-24T13:47:13,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456033996 2024-11-24T13:47:14,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033996 2024-11-24T13:47:14,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456033996) roll requested 2024-11-24T13:47:14,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034001 2024-11-24T13:47:14,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456033996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034001 2024-11-24T13:47:14,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034001) roll requested 2024-11-24T13:47:14,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034009 2024-11-24T13:47:14,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034009 2024-11-24T13:47:14,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034009) roll requested 2024-11-24T13:47:14,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034018 2024-11-24T13:47:14,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034018 2024-11-24T13:47:14,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034018) roll requested 2024-11-24T13:47:14,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034026 2024-11-24T13:47:14,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034026 2024-11-24T13:47:14,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034026) roll requested 2024-11-24T13:47:14,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034033 2024-11-24T13:47:14,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034033 2024-11-24T13:47:14,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034033) roll requested 2024-11-24T13:47:14,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034041 2024-11-24T13:47:14,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034041 2024-11-24T13:47:14,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034041) roll requested 2024-11-24T13:47:14,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034051 2024-11-24T13:47:14,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034051 2024-11-24T13:47:14,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,060 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034051) roll requested 2024-11-24T13:47:14,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034061 2024-11-24T13:47:14,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034061 2024-11-24T13:47:14,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034061) roll requested 2024-11-24T13:47:14,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034070 2024-11-24T13:47:14,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034070 2024-11-24T13:47:14,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,079 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034070) roll requested 2024-11-24T13:47:14,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034079 2024-11-24T13:47:14,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034079 2024-11-24T13:47:14,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034079) roll requested 2024-11-24T13:47:14,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034085 2024-11-24T13:47:14,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034085 2024-11-24T13:47:14,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034085) roll requested 2024-11-24T13:47:14,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034094 2024-11-24T13:47:14,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034094 2024-11-24T13:47:14,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,102 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034094) roll requested 2024-11-24T13:47:14,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034102 2024-11-24T13:47:14,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034102 2024-11-24T13:47:14,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034102) roll requested 2024-11-24T13:47:14,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034108 2024-11-24T13:47:14,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034102 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034108 2024-11-24T13:47:14,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034108) roll requested 2024-11-24T13:47:14,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034116 2024-11-24T13:47:14,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034116 2024-11-24T13:47:14,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034116) roll requested 2024-11-24T13:47:14,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034125 2024-11-24T13:47:14,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034125 2024-11-24T13:47:14,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034125) roll requested 2024-11-24T13:47:14,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034135 2024-11-24T13:47:14,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034135 2024-11-24T13:47:14,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034135) roll requested 2024-11-24T13:47:14,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034143 2024-11-24T13:47:14,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034143 2024-11-24T13:47:14,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034143) roll requested 2024-11-24T13:47:14,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034150 2024-11-24T13:47:14,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034150 2024-11-24T13:47:14,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034150) roll requested 2024-11-24T13:47:14,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034159 2024-11-24T13:47:14,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034159 2024-11-24T13:47:14,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034159) roll requested 2024-11-24T13:47:14,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034169 2024-11-24T13:47:14,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034169 2024-11-24T13:47:14,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034169) roll requested 2024-11-24T13:47:14,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034179 2024-11-24T13:47:14,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034179 2024-11-24T13:47:14,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,187 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034179) roll requested 2024-11-24T13:47:14,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034188 2024-11-24T13:47:14,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034188 2024-11-24T13:47:14,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034188) roll requested 2024-11-24T13:47:14,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034197 2024-11-24T13:47:14,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034197 2024-11-24T13:47:14,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034197) roll requested 2024-11-24T13:47:14,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034207 2024-11-24T13:47:14,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034207 2024-11-24T13:47:14,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,217 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034207) roll requested 2024-11-24T13:47:14,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034217 2024-11-24T13:47:14,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034217 2024-11-24T13:47:14,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034217) roll requested 2024-11-24T13:47:14,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034226 2024-11-24T13:47:14,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034217 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034226 2024-11-24T13:47:14,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,234 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034226) roll requested 2024-11-24T13:47:14,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034235 2024-11-24T13:47:14,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034235 2024-11-24T13:47:14,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034235) roll requested 2024-11-24T13:47:14,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034241 2024-11-24T13:47:14,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034235 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034241 2024-11-24T13:47:14,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034241) roll requested 2024-11-24T13:47:14,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034255 2024-11-24T13:47:14,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034255 2024-11-24T13:47:14,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034255) roll requested 2024-11-24T13:47:14,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034261 2024-11-24T13:47:14,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034261 2024-11-24T13:47:14,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,266 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034261) roll requested 2024-11-24T13:47:14,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034266 2024-11-24T13:47:14,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034266 2024-11-24T13:47:14,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034266) roll requested 2024-11-24T13:47:14,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034272 2024-11-24T13:47:14,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034272 2024-11-24T13:47:14,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034272) roll requested 2024-11-24T13:47:14,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034280 2024-11-24T13:47:14,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034280 2024-11-24T13:47:14,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034280) roll requested 2024-11-24T13:47:14,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034288 2024-11-24T13:47:14,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034288 2024-11-24T13:47:14,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034288) roll requested 2024-11-24T13:47:14,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034297 2024-11-24T13:47:14,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034297 2024-11-24T13:47:14,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034297) roll requested 2024-11-24T13:47:14,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034305 2024-11-24T13:47:14,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034305 2024-11-24T13:47:14,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034305) roll requested 2024-11-24T13:47:14,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034312 2024-11-24T13:47:14,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034312 2024-11-24T13:47:14,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034312) roll requested 2024-11-24T13:47:14,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034316 2024-11-24T13:47:14,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034316 2024-11-24T13:47:14,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,323 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034316) roll requested 2024-11-24T13:47:14,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034323 2024-11-24T13:47:14,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034316 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034323 2024-11-24T13:47:14,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034323) roll requested 2024-11-24T13:47:14,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034331 2024-11-24T13:47:14,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034331 2024-11-24T13:47:14,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034331) roll requested 2024-11-24T13:47:14,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034339 2024-11-24T13:47:14,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034339 2024-11-24T13:47:14,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034339) roll requested 2024-11-24T13:47:14,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034345 2024-11-24T13:47:14,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034345 2024-11-24T13:47:14,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034345) roll requested 2024-11-24T13:47:14,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034351 2024-11-24T13:47:14,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034351 2024-11-24T13:47:14,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,358 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034351) roll requested 2024-11-24T13:47:14,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034359 2024-11-24T13:47:14,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034351 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034359 2024-11-24T13:47:14,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034359) roll requested 2024-11-24T13:47:14,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034366 2024-11-24T13:47:14,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034366 2024-11-24T13:47:14,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034366) roll requested 2024-11-24T13:47:14,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034374 2024-11-24T13:47:14,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034374 2024-11-24T13:47:14,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,381 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034374) roll requested 2024-11-24T13:47:14,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034381 2024-11-24T13:47:14,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034381 2024-11-24T13:47:14,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,388 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034381) roll requested 2024-11-24T13:47:14,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034388 2024-11-24T13:47:14,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034381 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034388 2024-11-24T13:47:14,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034388) roll requested 2024-11-24T13:47:14,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034396 2024-11-24T13:47:14,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034396 2024-11-24T13:47:14,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034396) roll requested 2024-11-24T13:47:14,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034403 2024-11-24T13:47:14,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034403 2024-11-24T13:47:14,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034403) roll requested 2024-11-24T13:47:14,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034415 2024-11-24T13:47:14,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034415 2024-11-24T13:47:14,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034415) roll requested 2024-11-24T13:47:14,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034427 2024-11-24T13:47:14,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034427 2024-11-24T13:47:14,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034427) roll requested 2024-11-24T13:47:14,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034432 2024-11-24T13:47:14,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034432 2024-11-24T13:47:14,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034432) roll requested 2024-11-24T13:47:14,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034437 2024-11-24T13:47:14,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034437 2024-11-24T13:47:14,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,442 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034437) roll requested 2024-11-24T13:47:14,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034442 2024-11-24T13:47:14,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034442 2024-11-24T13:47:14,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034442) roll requested 2024-11-24T13:47:14,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034447 2024-11-24T13:47:14,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034447 2024-11-24T13:47:14,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034447) roll requested 2024-11-24T13:47:14,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034454 2024-11-24T13:47:14,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034454 2024-11-24T13:47:14,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034454) roll requested 2024-11-24T13:47:14,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034461 2024-11-24T13:47:14,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034461 2024-11-24T13:47:14,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,470 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034461) roll requested 2024-11-24T13:47:14,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034470 2024-11-24T13:47:14,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034470 2024-11-24T13:47:14,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034470) roll requested 2024-11-24T13:47:14,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034479 2024-11-24T13:47:14,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034470 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034479 2024-11-24T13:47:14,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034479) roll requested 2024-11-24T13:47:14,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034486 2024-11-24T13:47:14,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034486 2024-11-24T13:47:14,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034486) roll requested 2024-11-24T13:47:14,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034496 2024-11-24T13:47:14,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034496 2024-11-24T13:47:14,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034496) roll requested 2024-11-24T13:47:14,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034501 2024-11-24T13:47:14,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034501 2024-11-24T13:47:14,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034501) roll requested 2024-11-24T13:47:14,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034506 2024-11-24T13:47:14,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034506 2024-11-24T13:47:14,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034506) roll requested 2024-11-24T13:47:14,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034512 2024-11-24T13:47:14,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034512 2024-11-24T13:47:14,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,518 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034512) roll requested 2024-11-24T13:47:14,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034519 2024-11-24T13:47:14,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034519 2024-11-24T13:47:14,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034519) roll requested 2024-11-24T13:47:14,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034526 2024-11-24T13:47:14,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034526 2024-11-24T13:47:14,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034526) roll requested 2024-11-24T13:47:14,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034534 2024-11-24T13:47:14,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034534 2024-11-24T13:47:14,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,545 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034534) roll requested 2024-11-24T13:47:14,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034545 2024-11-24T13:47:14,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034545 2024-11-24T13:47:14,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034545) roll requested 2024-11-24T13:47:14,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034553 2024-11-24T13:47:14,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034553 2024-11-24T13:47:14,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034553) roll requested 2024-11-24T13:47:14,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034563 2024-11-24T13:47:14,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034563 2024-11-24T13:47:14,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034563) roll requested 2024-11-24T13:47:14,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034571 2024-11-24T13:47:14,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034571 2024-11-24T13:47:14,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034571) roll requested 2024-11-24T13:47:14,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034580 2024-11-24T13:47:14,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034580 2024-11-24T13:47:14,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034580) roll requested 2024-11-24T13:47:14,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034588 2024-11-24T13:47:14,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034588 2024-11-24T13:47:14,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034588) roll requested 2024-11-24T13:47:14,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034594 2024-11-24T13:47:14,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034594 2024-11-24T13:47:14,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034594) roll requested 2024-11-24T13:47:14,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034601 2024-11-24T13:47:14,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034601 2024-11-24T13:47:14,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034601) roll requested 2024-11-24T13:47:14,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034609 2024-11-24T13:47:14,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034609 2024-11-24T13:47:14,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034609) roll requested 2024-11-24T13:47:14,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034618 2024-11-24T13:47:14,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034618 2024-11-24T13:47:14,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,626 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034618) roll requested 2024-11-24T13:47:14,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034626 2024-11-24T13:47:14,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034626 2024-11-24T13:47:14,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034626) roll requested 2024-11-24T13:47:14,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034631 2024-11-24T13:47:14,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034626 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034631 2024-11-24T13:47:14,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034631) roll requested 2024-11-24T13:47:14,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034637 2024-11-24T13:47:14,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034637 2024-11-24T13:47:14,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034637) roll requested 2024-11-24T13:47:14,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034642 2024-11-24T13:47:14,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034642 2024-11-24T13:47:14,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034642) roll requested 2024-11-24T13:47:14,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034648 2024-11-24T13:47:14,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034642 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034648 2024-11-24T13:47:14,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034648) roll requested 2024-11-24T13:47:14,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034655 2024-11-24T13:47:14,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034655 2024-11-24T13:47:14,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034655) roll requested 2024-11-24T13:47:14,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034663 2024-11-24T13:47:14,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034663 2024-11-24T13:47:14,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,671 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034663) roll requested 2024-11-24T13:47:14,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034671 2024-11-24T13:47:14,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034663 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034671 2024-11-24T13:47:14,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034671) roll requested 2024-11-24T13:47:14,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034678 2024-11-24T13:47:14,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034671 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034678 2024-11-24T13:47:14,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034678) roll requested 2024-11-24T13:47:14,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034686 2024-11-24T13:47:14,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034686 2024-11-24T13:47:14,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034686) roll requested 2024-11-24T13:47:14,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034696 2024-11-24T13:47:14,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034696 2024-11-24T13:47:14,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034696) roll requested 2024-11-24T13:47:14,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034704 2024-11-24T13:47:14,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034704 2024-11-24T13:47:14,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034704) roll requested 2024-11-24T13:47:14,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034713 2024-11-24T13:47:14,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034713 2024-11-24T13:47:14,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034713) roll requested 2024-11-24T13:47:14,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034727 2024-11-24T13:47:14,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034727 2024-11-24T13:47:14,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034727) roll requested 2024-11-24T13:47:14,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034734 2024-11-24T13:47:14,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034734 2024-11-24T13:47:14,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,738 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034734) roll requested 2024-11-24T13:47:14,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034738 2024-11-24T13:47:14,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034738 2024-11-24T13:47:14,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034738) roll requested 2024-11-24T13:47:14,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034744 2024-11-24T13:47:14,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034738 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034744 2024-11-24T13:47:14,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034744) roll requested 2024-11-24T13:47:14,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034749 2024-11-24T13:47:14,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034749 2024-11-24T13:47:14,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034749) roll requested 2024-11-24T13:47:14,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034755 2024-11-24T13:47:14,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034755 2024-11-24T13:47:14,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,762 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034755) roll requested 2024-11-24T13:47:14,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034762 2024-11-24T13:47:14,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034762 2024-11-24T13:47:14,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034762) roll requested 2024-11-24T13:47:14,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034771 2024-11-24T13:47:14,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034762 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034771 2024-11-24T13:47:14,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034771) roll requested 2024-11-24T13:47:14,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034780 2024-11-24T13:47:14,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034780 2024-11-24T13:47:14,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,794 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034780) roll requested 2024-11-24T13:47:14,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034795 2024-11-24T13:47:14,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034795 2024-11-24T13:47:14,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034795) roll requested 2024-11-24T13:47:14,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034804 2024-11-24T13:47:14,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034804 2024-11-24T13:47:14,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034804) roll requested 2024-11-24T13:47:14,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034809 2024-11-24T13:47:14,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034809 2024-11-24T13:47:14,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034809) roll requested 2024-11-24T13:47:14,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034814 2024-11-24T13:47:14,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034809 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034814 2024-11-24T13:47:14,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034814) roll requested 2024-11-24T13:47:14,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034820 2024-11-24T13:47:14,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034820 2024-11-24T13:47:14,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034820) roll requested 2024-11-24T13:47:14,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034827 2024-11-24T13:47:14,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034827 2024-11-24T13:47:14,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034827) roll requested 2024-11-24T13:47:14,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034835 2024-11-24T13:47:14,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034835 2024-11-24T13:47:14,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034835) roll requested 2024-11-24T13:47:14,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034841 2024-11-24T13:47:14,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034835 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034841 2024-11-24T13:47:14,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034841) roll requested 2024-11-24T13:47:14,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034848 2024-11-24T13:47:14,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034848 2024-11-24T13:47:14,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034848) roll requested 2024-11-24T13:47:14,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034856 2024-11-24T13:47:14,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034856 2024-11-24T13:47:14,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034856) roll requested 2024-11-24T13:47:14,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034862 2024-11-24T13:47:14,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034862 2024-11-24T13:47:14,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034862) roll requested 2024-11-24T13:47:14,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034869 2024-11-24T13:47:14,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034869 2024-11-24T13:47:14,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034869) roll requested 2024-11-24T13:47:14,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034878 2024-11-24T13:47:14,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034878 2024-11-24T13:47:14,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,886 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034878) roll requested 2024-11-24T13:47:14,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034886 2024-11-24T13:47:14,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034886 2024-11-24T13:47:14,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034886) roll requested 2024-11-24T13:47:14,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034896 2024-11-24T13:47:14,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034886 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034896 2024-11-24T13:47:14,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034896) roll requested 2024-11-24T13:47:14,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034906 2024-11-24T13:47:14,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034906 2024-11-24T13:47:14,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034906) roll requested 2024-11-24T13:47:14,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034917 2024-11-24T13:47:14,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034917 2024-11-24T13:47:14,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034917) roll requested 2024-11-24T13:47:14,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034924 2024-11-24T13:47:14,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034924 2024-11-24T13:47:14,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034924) roll requested 2024-11-24T13:47:14,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034932 2024-11-24T13:47:14,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034932 2024-11-24T13:47:14,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034932) roll requested 2024-11-24T13:47:14,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034939 2024-11-24T13:47:14,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034939 2024-11-24T13:47:14,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034939) roll requested 2024-11-24T13:47:14,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034947 2024-11-24T13:47:14,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034947 2024-11-24T13:47:14,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034947) roll requested 2024-11-24T13:47:14,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034952 2024-11-24T13:47:14,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034952 2024-11-24T13:47:14,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034952) roll requested 2024-11-24T13:47:14,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034957 2024-11-24T13:47:14,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034957 2024-11-24T13:47:14,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034957) roll requested 2024-11-24T13:47:14,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034961 2024-11-24T13:47:14,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034957 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034961 2024-11-24T13:47:14,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034961) roll requested 2024-11-24T13:47:14,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034966 2024-11-24T13:47:14,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034966 2024-11-24T13:47:14,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034966) roll requested 2024-11-24T13:47:14,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034972 2024-11-24T13:47:14,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034972 2024-11-24T13:47:14,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034972) roll requested 2024-11-24T13:47:14,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034979 2024-11-24T13:47:14,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034979 2024-11-24T13:47:14,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034979) roll requested 2024-11-24T13:47:14,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034985 2024-11-24T13:47:14,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:14,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034979 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034985 2024-11-24T13:47:14,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:14,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:14,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034985) roll requested 2024-11-24T13:47:14,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456034993 2024-11-24T13:47:15,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034993 2024-11-24T13:47:15,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456034993) roll requested 2024-11-24T13:47:15,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035001 2024-11-24T13:47:15,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456034993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035001 2024-11-24T13:47:15,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035001) roll requested 2024-11-24T13:47:15,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035008 2024-11-24T13:47:15,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035008 2024-11-24T13:47:15,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035008) roll requested 2024-11-24T13:47:15,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035015 2024-11-24T13:47:15,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035015 2024-11-24T13:47:15,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035015) roll requested 2024-11-24T13:47:15,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035021 2024-11-24T13:47:15,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035015 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035021 2024-11-24T13:47:15,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,029 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035021) roll requested 2024-11-24T13:47:15,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035029 2024-11-24T13:47:15,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035029 2024-11-24T13:47:15,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035029) roll requested 2024-11-24T13:47:15,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035037 2024-11-24T13:47:15,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035037 2024-11-24T13:47:15,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035037) roll requested 2024-11-24T13:47:15,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035043 2024-11-24T13:47:15,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035043 2024-11-24T13:47:15,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,050 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035043) roll requested 2024-11-24T13:47:15,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035050 2024-11-24T13:47:15,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035050 2024-11-24T13:47:15,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035050) roll requested 2024-11-24T13:47:15,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035055 2024-11-24T13:47:15,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035050 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035055 2024-11-24T13:47:15,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035055) roll requested 2024-11-24T13:47:15,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035062 2024-11-24T13:47:15,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035062 2024-11-24T13:47:15,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035062) roll requested 2024-11-24T13:47:15,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035069 2024-11-24T13:47:15,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035069 2024-11-24T13:47:15,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035069) roll requested 2024-11-24T13:47:15,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035077 2024-11-24T13:47:15,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035077 2024-11-24T13:47:15,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035077) roll requested 2024-11-24T13:47:15,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035086 2024-11-24T13:47:15,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035086 2024-11-24T13:47:15,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,095 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035086) roll requested 2024-11-24T13:47:15,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035096 2024-11-24T13:47:15,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035096 2024-11-24T13:47:15,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035096) roll requested 2024-11-24T13:47:15,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035103 2024-11-24T13:47:15,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035103 2024-11-24T13:47:15,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035103) roll requested 2024-11-24T13:47:15,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035112 2024-11-24T13:47:15,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035112 2024-11-24T13:47:15,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035112) roll requested 2024-11-24T13:47:15,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035121 2024-11-24T13:47:15,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035121 2024-11-24T13:47:15,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035121) roll requested 2024-11-24T13:47:15,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035129 2024-11-24T13:47:15,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035129 2024-11-24T13:47:15,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,139 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035129) roll requested 2024-11-24T13:47:15,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035139 2024-11-24T13:47:15,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035139 2024-11-24T13:47:15,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035139) roll requested 2024-11-24T13:47:15,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035148 2024-11-24T13:47:15,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035148 2024-11-24T13:47:15,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,155 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035148) roll requested 2024-11-24T13:47:15,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035156 2024-11-24T13:47:15,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035156 2024-11-24T13:47:15,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035156) roll requested 2024-11-24T13:47:15,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035165 2024-11-24T13:47:15,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035165 2024-11-24T13:47:15,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035165) roll requested 2024-11-24T13:47:15,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035170 2024-11-24T13:47:15,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035170 2024-11-24T13:47:15,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035170) roll requested 2024-11-24T13:47:15,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035175 2024-11-24T13:47:15,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035175 2024-11-24T13:47:15,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035175) roll requested 2024-11-24T13:47:15,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035179 2024-11-24T13:47:15,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035179 2024-11-24T13:47:15,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035179) roll requested 2024-11-24T13:47:15,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035185 2024-11-24T13:47:15,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035185 2024-11-24T13:47:15,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035185) roll requested 2024-11-24T13:47:15,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035191 2024-11-24T13:47:15,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035191 2024-11-24T13:47:15,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035191) roll requested 2024-11-24T13:47:15,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035197 2024-11-24T13:47:15,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035197 2024-11-24T13:47:15,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035197) roll requested 2024-11-24T13:47:15,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035204 2024-11-24T13:47:15,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035204 2024-11-24T13:47:15,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035204) roll requested 2024-11-24T13:47:15,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035212 2024-11-24T13:47:15,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035212 2024-11-24T13:47:15,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035212) roll requested 2024-11-24T13:47:15,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035218 2024-11-24T13:47:15,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035218 2024-11-24T13:47:15,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035218) roll requested 2024-11-24T13:47:15,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035226 2024-11-24T13:47:15,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035218 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035226 2024-11-24T13:47:15,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,234 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035226) roll requested 2024-11-24T13:47:15,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035235 2024-11-24T13:47:15,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035235 2024-11-24T13:47:15,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035235) roll requested 2024-11-24T13:47:15,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035244 2024-11-24T13:47:15,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035235 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035244 2024-11-24T13:47:15,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035244) roll requested 2024-11-24T13:47:15,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035253 2024-11-24T13:47:15,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035253 2024-11-24T13:47:15,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035253) roll requested 2024-11-24T13:47:15,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035262 2024-11-24T13:47:15,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035262 2024-11-24T13:47:15,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035262) roll requested 2024-11-24T13:47:15,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035271 2024-11-24T13:47:15,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035271 2024-11-24T13:47:15,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035271) roll requested 2024-11-24T13:47:15,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035279 2024-11-24T13:47:15,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035279 2024-11-24T13:47:15,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035279) roll requested 2024-11-24T13:47:15,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035287 2024-11-24T13:47:15,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035287 2024-11-24T13:47:15,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035287) roll requested 2024-11-24T13:47:15,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035296 2024-11-24T13:47:15,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035296 2024-11-24T13:47:15,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035296) roll requested 2024-11-24T13:47:15,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035301 2024-11-24T13:47:15,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035296 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035301 2024-11-24T13:47:15,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,307 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035301) roll requested 2024-11-24T13:47:15,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035307 2024-11-24T13:47:15,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035307 2024-11-24T13:47:15,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035307) roll requested 2024-11-24T13:47:15,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035314 2024-11-24T13:47:15,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035314 2024-11-24T13:47:15,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035314) roll requested 2024-11-24T13:47:15,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035321 2024-11-24T13:47:15,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035321 2024-11-24T13:47:15,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035321) roll requested 2024-11-24T13:47:15,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035326 2024-11-24T13:47:15,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035326 2024-11-24T13:47:15,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035326) roll requested 2024-11-24T13:47:15,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035331 2024-11-24T13:47:15,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035331 2024-11-24T13:47:15,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035331) roll requested 2024-11-24T13:47:15,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035338 2024-11-24T13:47:15,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035338 2024-11-24T13:47:15,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035338) roll requested 2024-11-24T13:47:15,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035345 2024-11-24T13:47:15,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035338 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035345 2024-11-24T13:47:15,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035345) roll requested 2024-11-24T13:47:15,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035353 2024-11-24T13:47:15,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035353 2024-11-24T13:47:15,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035353) roll requested 2024-11-24T13:47:15,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035363 2024-11-24T13:47:15,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035363 2024-11-24T13:47:15,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035363) roll requested 2024-11-24T13:47:15,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035372 2024-11-24T13:47:15,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035372 2024-11-24T13:47:15,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035372) roll requested 2024-11-24T13:47:15,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035382 2024-11-24T13:47:15,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035382 2024-11-24T13:47:15,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035382) roll requested 2024-11-24T13:47:15,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035389 2024-11-24T13:47:15,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035389 2024-11-24T13:47:15,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035389) roll requested 2024-11-24T13:47:15,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035396 2024-11-24T13:47:15,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035396 2024-11-24T13:47:15,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035396) roll requested 2024-11-24T13:47:15,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035404 2024-11-24T13:47:15,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035404 2024-11-24T13:47:15,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035404) roll requested 2024-11-24T13:47:15,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035412 2024-11-24T13:47:15,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035412 2024-11-24T13:47:15,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035412) roll requested 2024-11-24T13:47:15,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035421 2024-11-24T13:47:15,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035421 2024-11-24T13:47:15,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035421) roll requested 2024-11-24T13:47:15,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035427 2024-11-24T13:47:15,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035427 2024-11-24T13:47:15,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035427) roll requested 2024-11-24T13:47:15,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035435 2024-11-24T13:47:15,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035435 2024-11-24T13:47:15,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,442 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035435) roll requested 2024-11-24T13:47:15,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035442 2024-11-24T13:47:15,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035442 2024-11-24T13:47:15,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035442) roll requested 2024-11-24T13:47:15,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035447 2024-11-24T13:47:15,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035447 2024-11-24T13:47:15,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035447) roll requested 2024-11-24T13:47:15,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035453 2024-11-24T13:47:15,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035453 2024-11-24T13:47:15,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035453) roll requested 2024-11-24T13:47:15,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035460 2024-11-24T13:47:15,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035460 2024-11-24T13:47:15,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035460) roll requested 2024-11-24T13:47:15,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035467 2024-11-24T13:47:15,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035467 2024-11-24T13:47:15,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035467) roll requested 2024-11-24T13:47:15,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035474 2024-11-24T13:47:15,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035474 2024-11-24T13:47:15,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035474) roll requested 2024-11-24T13:47:15,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035481 2024-11-24T13:47:15,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035481 2024-11-24T13:47:15,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035481) roll requested 2024-11-24T13:47:15,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035488 2024-11-24T13:47:15,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035488 2024-11-24T13:47:15,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035488) roll requested 2024-11-24T13:47:15,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035497 2024-11-24T13:47:15,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035497 2024-11-24T13:47:15,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035497) roll requested 2024-11-24T13:47:15,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035507 2024-11-24T13:47:15,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035507 2024-11-24T13:47:15,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035507) roll requested 2024-11-24T13:47:15,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035516 2024-11-24T13:47:15,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035516 2024-11-24T13:47:15,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,525 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035516) roll requested 2024-11-24T13:47:15,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035525 2024-11-24T13:47:15,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035516 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035525 2024-11-24T13:47:15,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035525) roll requested 2024-11-24T13:47:15,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035531 2024-11-24T13:47:15,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035531 2024-11-24T13:47:15,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035531) roll requested 2024-11-24T13:47:15,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035537 2024-11-24T13:47:15,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035537 2024-11-24T13:47:15,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035537) roll requested 2024-11-24T13:47:15,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035547 2024-11-24T13:47:15,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035547 2024-11-24T13:47:15,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035547) roll requested 2024-11-24T13:47:15,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035556 2024-11-24T13:47:15,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035556 2024-11-24T13:47:15,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035556) roll requested 2024-11-24T13:47:15,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035565 2024-11-24T13:47:15,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035565 2024-11-24T13:47:15,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035565) roll requested 2024-11-24T13:47:15,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035574 2024-11-24T13:47:15,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035574 2024-11-24T13:47:15,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035574) roll requested 2024-11-24T13:47:15,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035581 2024-11-24T13:47:15,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035574 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035581 2024-11-24T13:47:15,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035581) roll requested 2024-11-24T13:47:15,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035588 2024-11-24T13:47:15,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035588 2024-11-24T13:47:15,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035588) roll requested 2024-11-24T13:47:15,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035596 2024-11-24T13:47:15,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035596 2024-11-24T13:47:15,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035596) roll requested 2024-11-24T13:47:15,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035602 2024-11-24T13:47:15,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035602 2024-11-24T13:47:15,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035602) roll requested 2024-11-24T13:47:15,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035609 2024-11-24T13:47:15,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035609 2024-11-24T13:47:15,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,615 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035609) roll requested 2024-11-24T13:47:15,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035615 2024-11-24T13:47:15,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035615 2024-11-24T13:47:15,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035615) roll requested 2024-11-24T13:47:15,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035623 2024-11-24T13:47:15,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035615 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035623 2024-11-24T13:47:15,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035623) roll requested 2024-11-24T13:47:15,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035632 2024-11-24T13:47:15,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035632 2024-11-24T13:47:15,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035632) roll requested 2024-11-24T13:47:15,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035642 2024-11-24T13:47:15,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035642 2024-11-24T13:47:15,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035642) roll requested 2024-11-24T13:47:15,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035650 2024-11-24T13:47:15,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035642 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035650 2024-11-24T13:47:15,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035650) roll requested 2024-11-24T13:47:15,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035656 2024-11-24T13:47:15,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035656 2024-11-24T13:47:15,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035656) roll requested 2024-11-24T13:47:15,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035662 2024-11-24T13:47:15,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035662 2024-11-24T13:47:15,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035662) roll requested 2024-11-24T13:47:15,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035669 2024-11-24T13:47:15,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035669 2024-11-24T13:47:15,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035669) roll requested 2024-11-24T13:47:15,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035678 2024-11-24T13:47:15,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035678 2024-11-24T13:47:15,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035678) roll requested 2024-11-24T13:47:15,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035685 2024-11-24T13:47:15,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035685 2024-11-24T13:47:15,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035685) roll requested 2024-11-24T13:47:15,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035692 2024-11-24T13:47:15,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035692 2024-11-24T13:47:15,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035692) roll requested 2024-11-24T13:47:15,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035701 2024-11-24T13:47:15,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035701 2024-11-24T13:47:15,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035701) roll requested 2024-11-24T13:47:15,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035710 2024-11-24T13:47:15,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035710 2024-11-24T13:47:15,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035710) roll requested 2024-11-24T13:47:15,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035719 2024-11-24T13:47:15,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035710 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035719 2024-11-24T13:47:15,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035719) roll requested 2024-11-24T13:47:15,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035726 2024-11-24T13:47:15,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035726 2024-11-24T13:47:15,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035726) roll requested 2024-11-24T13:47:15,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035734 2024-11-24T13:47:15,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035734 2024-11-24T13:47:15,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035734) roll requested 2024-11-24T13:47:15,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035744 2024-11-24T13:47:15,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035744 2024-11-24T13:47:15,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035744) roll requested 2024-11-24T13:47:15,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035749 2024-11-24T13:47:15,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035749 2024-11-24T13:47:15,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,754 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035749) roll requested 2024-11-24T13:47:15,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035754 2024-11-24T13:47:15,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035754 2024-11-24T13:47:15,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035754) roll requested 2024-11-24T13:47:15,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035761 2024-11-24T13:47:15,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035754 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035761 2024-11-24T13:47:15,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035761) roll requested 2024-11-24T13:47:15,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035768 2024-11-24T13:47:15,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035768 2024-11-24T13:47:15,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035768) roll requested 2024-11-24T13:47:15,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035775 2024-11-24T13:47:15,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035775 2024-11-24T13:47:15,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035775) roll requested 2024-11-24T13:47:15,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035783 2024-11-24T13:47:15,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035783 2024-11-24T13:47:15,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035783) roll requested 2024-11-24T13:47:15,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035789 2024-11-24T13:47:15,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035789 2024-11-24T13:47:15,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035789) roll requested 2024-11-24T13:47:15,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035797 2024-11-24T13:47:15,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035797 2024-11-24T13:47:15,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035797) roll requested 2024-11-24T13:47:15,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035805 2024-11-24T13:47:15,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035805 2024-11-24T13:47:15,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035805) roll requested 2024-11-24T13:47:15,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035812 2024-11-24T13:47:15,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035812 2024-11-24T13:47:15,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,823 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035812) roll requested 2024-11-24T13:47:15,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035823 2024-11-24T13:47:15,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035823 2024-11-24T13:47:15,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,830 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035823) roll requested 2024-11-24T13:47:15,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035830 2024-11-24T13:47:15,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035823 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035830 2024-11-24T13:47:15,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035830) roll requested 2024-11-24T13:47:15,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035835 2024-11-24T13:47:15,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035830 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035835 2024-11-24T13:47:15,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035835) roll requested 2024-11-24T13:47:15,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035841 2024-11-24T13:47:15,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035835 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035841 2024-11-24T13:47:15,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,846 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035841) roll requested 2024-11-24T13:47:15,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035847 2024-11-24T13:47:15,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035847 2024-11-24T13:47:15,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035847) roll requested 2024-11-24T13:47:15,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035854 2024-11-24T13:47:15,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035854 2024-11-24T13:47:15,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035854) roll requested 2024-11-24T13:47:15,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035862 2024-11-24T13:47:15,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035862 2024-11-24T13:47:15,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,870 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035862) roll requested 2024-11-24T13:47:15,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035871 2024-11-24T13:47:15,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035871 2024-11-24T13:47:15,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035871) roll requested 2024-11-24T13:47:15,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035879 2024-11-24T13:47:15,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035879 2024-11-24T13:47:15,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035879) roll requested 2024-11-24T13:47:15,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035885 2024-11-24T13:47:15,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035879 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035885 2024-11-24T13:47:15,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,891 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035885) roll requested 2024-11-24T13:47:15,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035891 2024-11-24T13:47:15,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035891 2024-11-24T13:47:15,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035891) roll requested 2024-11-24T13:47:15,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035898 2024-11-24T13:47:15,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035898 2024-11-24T13:47:15,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035898) roll requested 2024-11-24T13:47:15,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035906 2024-11-24T13:47:15,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035906 2024-11-24T13:47:15,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035906) roll requested 2024-11-24T13:47:15,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035915 2024-11-24T13:47:15,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035915 2024-11-24T13:47:15,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,923 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035915) roll requested 2024-11-24T13:47:15,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035923 2024-11-24T13:47:15,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035923 2024-11-24T13:47:15,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035923) roll requested 2024-11-24T13:47:15,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035929 2024-11-24T13:47:15,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035923 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035929 2024-11-24T13:47:15,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035929) roll requested 2024-11-24T13:47:15,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035937 2024-11-24T13:47:15,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035937 2024-11-24T13:47:15,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035937) roll requested 2024-11-24T13:47:15,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035947 2024-11-24T13:47:15,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035947 2024-11-24T13:47:15,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035947) roll requested 2024-11-24T13:47:15,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035954 2024-11-24T13:47:15,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035954 2024-11-24T13:47:15,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,962 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035954) roll requested 2024-11-24T13:47:15,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035962 2024-11-24T13:47:15,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035962 2024-11-24T13:47:15,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,970 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035962) roll requested 2024-11-24T13:47:15,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035970 2024-11-24T13:47:15,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035962 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035970 2024-11-24T13:47:15,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035970) roll requested 2024-11-24T13:47:15,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035979 2024-11-24T13:47:15,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035970 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035979 2024-11-24T13:47:15,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035979) roll requested 2024-11-24T13:47:15,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035987 2024-11-24T13:47:15,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:15,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035979 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035987 2024-11-24T13:47:15,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:15,994 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:15,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035987) roll requested 2024-11-24T13:47:15,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456035994 2024-11-24T13:47:16,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035994 2024-11-24T13:47:16,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456035994) roll requested 2024-11-24T13:47:16,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036001 2024-11-24T13:47:16,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456035994 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036001 2024-11-24T13:47:16,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036001) roll requested 2024-11-24T13:47:16,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036010 2024-11-24T13:47:16,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036010 2024-11-24T13:47:16,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036010) roll requested 2024-11-24T13:47:16,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036017 2024-11-24T13:47:16,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036017 2024-11-24T13:47:16,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036017) roll requested 2024-11-24T13:47:16,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036026 2024-11-24T13:47:16,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036026 2024-11-24T13:47:16,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036026) roll requested 2024-11-24T13:47:16,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036035 2024-11-24T13:47:16,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036035 2024-11-24T13:47:16,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036035) roll requested 2024-11-24T13:47:16,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036044 2024-11-24T13:47:16,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036044 2024-11-24T13:47:16,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036044) roll requested 2024-11-24T13:47:16,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036053 2024-11-24T13:47:16,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036053 2024-11-24T13:47:16,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,058 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036053) roll requested 2024-11-24T13:47:16,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036058 2024-11-24T13:47:16,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036053 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036058 2024-11-24T13:47:16,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036058) roll requested 2024-11-24T13:47:16,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036064 2024-11-24T13:47:16,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036058 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036064 2024-11-24T13:47:16,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036064) roll requested 2024-11-24T13:47:16,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036070 2024-11-24T13:47:16,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036070 2024-11-24T13:47:16,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036070) roll requested 2024-11-24T13:47:16,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036077 2024-11-24T13:47:16,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036077 2024-11-24T13:47:16,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036077) roll requested 2024-11-24T13:47:16,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036085 2024-11-24T13:47:16,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036085 2024-11-24T13:47:16,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036085) roll requested 2024-11-24T13:47:16,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036092 2024-11-24T13:47:16,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036092 2024-11-24T13:47:16,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036092) roll requested 2024-11-24T13:47:16,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036099 2024-11-24T13:47:16,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036099 2024-11-24T13:47:16,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036099) roll requested 2024-11-24T13:47:16,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036107 2024-11-24T13:47:16,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036107 2024-11-24T13:47:16,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036107) roll requested 2024-11-24T13:47:16,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036116 2024-11-24T13:47:16,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036116 2024-11-24T13:47:16,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036116) roll requested 2024-11-24T13:47:16,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036125 2024-11-24T13:47:16,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036125 2024-11-24T13:47:16,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036125) roll requested 2024-11-24T13:47:16,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036135 2024-11-24T13:47:16,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036135 2024-11-24T13:47:16,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036135) roll requested 2024-11-24T13:47:16,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036143 2024-11-24T13:47:16,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036143 2024-11-24T13:47:16,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036143) roll requested 2024-11-24T13:47:16,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036153 2024-11-24T13:47:16,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036153 2024-11-24T13:47:16,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036153) roll requested 2024-11-24T13:47:16,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036160 2024-11-24T13:47:16,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036160 2024-11-24T13:47:16,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,166 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036160) roll requested 2024-11-24T13:47:16,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036166 2024-11-24T13:47:16,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036166 2024-11-24T13:47:16,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,174 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036166) roll requested 2024-11-24T13:47:16,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036174 2024-11-24T13:47:16,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036166 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036174 2024-11-24T13:47:16,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036174) roll requested 2024-11-24T13:47:16,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036183 2024-11-24T13:47:16,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036174 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036183 2024-11-24T13:47:16,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036183) roll requested 2024-11-24T13:47:16,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036191 2024-11-24T13:47:16,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036191 2024-11-24T13:47:16,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036191) roll requested 2024-11-24T13:47:16,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036201 2024-11-24T13:47:16,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036201 2024-11-24T13:47:16,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,209 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036201) roll requested 2024-11-24T13:47:16,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036210 2024-11-24T13:47:16,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036210 2024-11-24T13:47:16,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036210) roll requested 2024-11-24T13:47:16,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036220 2024-11-24T13:47:16,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036220 2024-11-24T13:47:16,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036220) roll requested 2024-11-24T13:47:16,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036227 2024-11-24T13:47:16,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036227 2024-11-24T13:47:16,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036227) roll requested 2024-11-24T13:47:16,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036233 2024-11-24T13:47:16,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036233 2024-11-24T13:47:16,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036233) roll requested 2024-11-24T13:47:16,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036237 2024-11-24T13:47:16,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036237 2024-11-24T13:47:16,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036237) roll requested 2024-11-24T13:47:16,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036242 2024-11-24T13:47:16,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036242 2024-11-24T13:47:16,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036242) roll requested 2024-11-24T13:47:16,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036247 2024-11-24T13:47:16,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036247 2024-11-24T13:47:16,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036247) roll requested 2024-11-24T13:47:16,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036254 2024-11-24T13:47:16,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036254 2024-11-24T13:47:16,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036254) roll requested 2024-11-24T13:47:16,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036261 2024-11-24T13:47:16,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036261 2024-11-24T13:47:16,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,269 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036261) roll requested 2024-11-24T13:47:16,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036269 2024-11-24T13:47:16,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036269 2024-11-24T13:47:16,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036269) roll requested 2024-11-24T13:47:16,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036277 2024-11-24T13:47:16,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036277 2024-11-24T13:47:16,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036277) roll requested 2024-11-24T13:47:16,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036285 2024-11-24T13:47:16,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036285 2024-11-24T13:47:16,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036285) roll requested 2024-11-24T13:47:16,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036295 2024-11-24T13:47:16,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036295 2024-11-24T13:47:16,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,303 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036295) roll requested 2024-11-24T13:47:16,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036304 2024-11-24T13:47:16,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036304 2024-11-24T13:47:16,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,309 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036304) roll requested 2024-11-24T13:47:16,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036309 2024-11-24T13:47:16,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036309 2024-11-24T13:47:16,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036309) roll requested 2024-11-24T13:47:16,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036314 2024-11-24T13:47:16,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036314 2024-11-24T13:47:16,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036314) roll requested 2024-11-24T13:47:16,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036320 2024-11-24T13:47:16,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036320 2024-11-24T13:47:16,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036320) roll requested 2024-11-24T13:47:16,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036327 2024-11-24T13:47:16,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036327 2024-11-24T13:47:16,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,333 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036327) roll requested 2024-11-24T13:47:16,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036333 2024-11-24T13:47:16,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036333 2024-11-24T13:47:16,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036333) roll requested 2024-11-24T13:47:16,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036339 2024-11-24T13:47:16,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036339 2024-11-24T13:47:16,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036339) roll requested 2024-11-24T13:47:16,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036346 2024-11-24T13:47:16,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036346 2024-11-24T13:47:16,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036346) roll requested 2024-11-24T13:47:16,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036355 2024-11-24T13:47:16,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036355 2024-11-24T13:47:16,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036355) roll requested 2024-11-24T13:47:16,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036364 2024-11-24T13:47:16,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036364 2024-11-24T13:47:16,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036364) roll requested 2024-11-24T13:47:16,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036373 2024-11-24T13:47:16,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036373 2024-11-24T13:47:16,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036373) roll requested 2024-11-24T13:47:16,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036381 2024-11-24T13:47:16,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036373 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036381 2024-11-24T13:47:16,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036381) roll requested 2024-11-24T13:47:16,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036389 2024-11-24T13:47:16,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036381 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036389 2024-11-24T13:47:16,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,398 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036389) roll requested 2024-11-24T13:47:16,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036399 2024-11-24T13:47:16,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036399 2024-11-24T13:47:16,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036399) roll requested 2024-11-24T13:47:16,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036406 2024-11-24T13:47:16,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036406 2024-11-24T13:47:16,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036406) roll requested 2024-11-24T13:47:16,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036414 2024-11-24T13:47:16,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036414 2024-11-24T13:47:16,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036414) roll requested 2024-11-24T13:47:16,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036422 2024-11-24T13:47:16,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036422 2024-11-24T13:47:16,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036422) roll requested 2024-11-24T13:47:16,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036432 2024-11-24T13:47:16,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036432 2024-11-24T13:47:16,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036432) roll requested 2024-11-24T13:47:16,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036441 2024-11-24T13:47:16,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036441 2024-11-24T13:47:16,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036441) roll requested 2024-11-24T13:47:16,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036451 2024-11-24T13:47:16,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036451 2024-11-24T13:47:16,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036451) roll requested 2024-11-24T13:47:16,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036459 2024-11-24T13:47:16,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036459 2024-11-24T13:47:16,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036459) roll requested 2024-11-24T13:47:16,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036466 2024-11-24T13:47:16,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036466 2024-11-24T13:47:16,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036466) roll requested 2024-11-24T13:47:16,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036474 2024-11-24T13:47:16,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036474 2024-11-24T13:47:16,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036474) roll requested 2024-11-24T13:47:16,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036482 2024-11-24T13:47:16,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036482 2024-11-24T13:47:16,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036482) roll requested 2024-11-24T13:47:16,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036490 2024-11-24T13:47:16,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036482 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036490 2024-11-24T13:47:16,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036490) roll requested 2024-11-24T13:47:16,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036500 2024-11-24T13:47:16,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036500 2024-11-24T13:47:16,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036500) roll requested 2024-11-24T13:47:16,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036510 2024-11-24T13:47:16,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036510 2024-11-24T13:47:16,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,518 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036510) roll requested 2024-11-24T13:47:16,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036519 2024-11-24T13:47:16,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036510 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036519 2024-11-24T13:47:16,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036519) roll requested 2024-11-24T13:47:16,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036526 2024-11-24T13:47:16,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036526 2024-11-24T13:47:16,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036526) roll requested 2024-11-24T13:47:16,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036533 2024-11-24T13:47:16,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036533 2024-11-24T13:47:16,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036533) roll requested 2024-11-24T13:47:16,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036540 2024-11-24T13:47:16,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036540 2024-11-24T13:47:16,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036540) roll requested 2024-11-24T13:47:16,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036549 2024-11-24T13:47:16,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036540 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036549 2024-11-24T13:47:16,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036549) roll requested 2024-11-24T13:47:16,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036558 2024-11-24T13:47:16,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036558 2024-11-24T13:47:16,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036558) roll requested 2024-11-24T13:47:16,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036569 2024-11-24T13:47:16,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036569 2024-11-24T13:47:16,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036569) roll requested 2024-11-24T13:47:16,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036581 2024-11-24T13:47:16,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036581 2024-11-24T13:47:16,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036581) roll requested 2024-11-24T13:47:16,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036586 2024-11-24T13:47:16,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036586 2024-11-24T13:47:16,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,590 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036586) roll requested 2024-11-24T13:47:16,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036591 2024-11-24T13:47:16,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036586 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036591 2024-11-24T13:47:16,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036591) roll requested 2024-11-24T13:47:16,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036597 2024-11-24T13:47:16,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036591 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036597 2024-11-24T13:47:16,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036597) roll requested 2024-11-24T13:47:16,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036603 2024-11-24T13:47:16,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036603 2024-11-24T13:47:16,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036603) roll requested 2024-11-24T13:47:16,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036611 2024-11-24T13:47:16,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036611 2024-11-24T13:47:16,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036611) roll requested 2024-11-24T13:47:16,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036621 2024-11-24T13:47:16,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036611 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036621 2024-11-24T13:47:16,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,630 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036621) roll requested 2024-11-24T13:47:16,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036630 2024-11-24T13:47:16,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036630 2024-11-24T13:47:16,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036630) roll requested 2024-11-24T13:47:16,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036640 2024-11-24T13:47:16,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036630 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036640 2024-11-24T13:47:16,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036640) roll requested 2024-11-24T13:47:16,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036646 2024-11-24T13:47:16,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036646 2024-11-24T13:47:16,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036646) roll requested 2024-11-24T13:47:16,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036654 2024-11-24T13:47:16,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036646 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036654 2024-11-24T13:47:16,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,661 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036654) roll requested 2024-11-24T13:47:16,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036661 2024-11-24T13:47:16,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036661 2024-11-24T13:47:16,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036661) roll requested 2024-11-24T13:47:16,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036670 2024-11-24T13:47:16,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036670 2024-11-24T13:47:16,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,678 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036670) roll requested 2024-11-24T13:47:16,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036678 2024-11-24T13:47:16,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036678 2024-11-24T13:47:16,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:16,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036678) roll requested 2024-11-24T13:47:16,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036691 2024-11-24T13:47:16,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036691 2024-11-24T13:47:16,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,700 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036691) roll requested 2024-11-24T13:47:16,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036701 2024-11-24T13:47:16,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036691 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036701 2024-11-24T13:47:16,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,707 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036701) roll requested 2024-11-24T13:47:16,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036707 2024-11-24T13:47:16,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036707 2024-11-24T13:47:16,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,714 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036707) roll requested 2024-11-24T13:47:16,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036714 2024-11-24T13:47:16,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036707 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036714 2024-11-24T13:47:16,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,723 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036714) roll requested 2024-11-24T13:47:16,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036723 2024-11-24T13:47:16,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036714 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036723 2024-11-24T13:47:16,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,733 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036723) roll requested 2024-11-24T13:47:16,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036733 2024-11-24T13:47:16,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036723 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036733 2024-11-24T13:47:16,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,743 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036733) roll requested 2024-11-24T13:47:16,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036743 2024-11-24T13:47:16,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036733 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036743 2024-11-24T13:47:16,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,751 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036743) roll requested 2024-11-24T13:47:16,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036752 2024-11-24T13:47:16,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036743 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036752 2024-11-24T13:47:16,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,759 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036752) roll requested 2024-11-24T13:47:16,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036759 2024-11-24T13:47:16,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036752 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036759 2024-11-24T13:47:16,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,768 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036759) roll requested 2024-11-24T13:47:16,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036768 2024-11-24T13:47:16,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036759 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036768 2024-11-24T13:47:16,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,774 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036768) roll requested 2024-11-24T13:47:16,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036774 2024-11-24T13:47:16,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036768 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036774 2024-11-24T13:47:16,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,783 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036774) roll requested 2024-11-24T13:47:16,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036783 2024-11-24T13:47:16,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036774 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036783 2024-11-24T13:47:16,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,788 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036783) roll requested 2024-11-24T13:47:16,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036788 2024-11-24T13:47:16,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036783 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036788 2024-11-24T13:47:16,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,795 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036788) roll requested 2024-11-24T13:47:16,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036795 2024-11-24T13:47:16,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036788 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036795 2024-11-24T13:47:16,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,800 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036795) roll requested 2024-11-24T13:47:16,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036800 2024-11-24T13:47:16,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036795 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036800 2024-11-24T13:47:16,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,805 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036800) roll requested 2024-11-24T13:47:16,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036805 2024-11-24T13:47:16,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036800 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036805 2024-11-24T13:47:16,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,810 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036805) roll requested 2024-11-24T13:47:16,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036811 2024-11-24T13:47:16,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036805 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036811 2024-11-24T13:47:16,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,817 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036811) roll requested 2024-11-24T13:47:16,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036817 2024-11-24T13:47:16,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036811 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036817 2024-11-24T13:47:16,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,826 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036817) roll requested 2024-11-24T13:47:16,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036826 2024-11-24T13:47:16,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036817 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036826 2024-11-24T13:47:16,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,835 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036826) roll requested 2024-11-24T13:47:16,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036835 2024-11-24T13:47:16,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036826 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036835 2024-11-24T13:47:16,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,841 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036835) roll requested 2024-11-24T13:47:16,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036841 2024-11-24T13:47:16,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036835 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036841 2024-11-24T13:47:16,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,846 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036841) roll requested 2024-11-24T13:47:16,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036846 2024-11-24T13:47:16,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036841 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036846 2024-11-24T13:47:16,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,852 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036846) roll requested 2024-11-24T13:47:16,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036852 2024-11-24T13:47:16,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036846 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036852 2024-11-24T13:47:16,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,859 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036852) roll requested 2024-11-24T13:47:16,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036859 2024-11-24T13:47:16,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036852 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036859 2024-11-24T13:47:16,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,866 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036859) roll requested 2024-11-24T13:47:16,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036866 2024-11-24T13:47:16,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036859 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036866 2024-11-24T13:47:16,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,875 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036866) roll requested 2024-11-24T13:47:16,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036875 2024-11-24T13:47:16,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036866 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036875 2024-11-24T13:47:16,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,882 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036875) roll requested 2024-11-24T13:47:16,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036882 2024-11-24T13:47:16,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036875 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036882 2024-11-24T13:47:16,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,887 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036882) roll requested 2024-11-24T13:47:16,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036888 2024-11-24T13:47:16,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036882 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036888 2024-11-24T13:47:16,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,895 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036888) roll requested 2024-11-24T13:47:16,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036895 2024-11-24T13:47:16,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036888 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036895 2024-11-24T13:47:16,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,900 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036895) roll requested 2024-11-24T13:47:16,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036900 2024-11-24T13:47:16,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036895 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036900 2024-11-24T13:47:16,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,906 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036900) roll requested 2024-11-24T13:47:16,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036906 2024-11-24T13:47:16,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036900 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036906 2024-11-24T13:47:16,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,911 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036906) roll requested 2024-11-24T13:47:16,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036911 2024-11-24T13:47:16,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036906 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036911 2024-11-24T13:47:16,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,916 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036911) roll requested 2024-11-24T13:47:16,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036917 2024-11-24T13:47:16,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036911 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036917 2024-11-24T13:47:16,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,922 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036917) roll requested 2024-11-24T13:47:16,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036922 2024-11-24T13:47:16,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036917 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036922 2024-11-24T13:47:16,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,928 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036922) roll requested 2024-11-24T13:47:16,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036928 2024-11-24T13:47:16,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036922 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036928 2024-11-24T13:47:16,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,936 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036928) roll requested 2024-11-24T13:47:16,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036936 2024-11-24T13:47:16,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036928 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036936 2024-11-24T13:47:16,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,944 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036936) roll requested 2024-11-24T13:47:16,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036945 2024-11-24T13:47:16,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036936 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036945 2024-11-24T13:47:16,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,954 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036945) roll requested 2024-11-24T13:47:16,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036954 2024-11-24T13:47:16,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036945 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036954 2024-11-24T13:47:16,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,961 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036954) roll requested 2024-11-24T13:47:16,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036961 2024-11-24T13:47:16,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036954 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036961 2024-11-24T13:47:16,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,969 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036961) roll requested 2024-11-24T13:47:16,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036969 2024-11-24T13:47:16,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036961 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036969 2024-11-24T13:47:16,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,974 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036969) roll requested 2024-11-24T13:47:16,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036975 2024-11-24T13:47:16,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036969 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036975 2024-11-24T13:47:16,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,980 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036975) roll requested 2024-11-24T13:47:16,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036980 2024-11-24T13:47:16,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036975 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036980 2024-11-24T13:47:16,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,985 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036980) roll requested 2024-11-24T13:47:16,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036986 2024-11-24T13:47:16,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036980 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036986 2024-11-24T13:47:16,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:16,992 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:16,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036986) roll requested 2024-11-24T13:47:16,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456036993 2024-11-24T13:47:16,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:16,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036986 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036993 2024-11-24T13:47:17,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,000 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456036993) roll requested 2024-11-24T13:47:17,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037000 2024-11-24T13:47:17,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456036993 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037000 2024-11-24T13:47:17,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,008 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037000) roll requested 2024-11-24T13:47:17,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037009 2024-11-24T13:47:17,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037000 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037009 2024-11-24T13:47:17,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,017 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037009) roll requested 2024-11-24T13:47:17,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037017 2024-11-24T13:47:17,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037009 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037017 2024-11-24T13:47:17,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,025 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037017) roll requested 2024-11-24T13:47:17,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037026 2024-11-24T13:47:17,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037017 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037026 2024-11-24T13:47:17,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,034 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037026) roll requested 2024-11-24T13:47:17,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037035 2024-11-24T13:47:17,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037026 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037035 2024-11-24T13:47:17,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,044 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037035) roll requested 2024-11-24T13:47:17,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037045 2024-11-24T13:47:17,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037035 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037045 2024-11-24T13:47:17,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,053 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037045) roll requested 2024-11-24T13:47:17,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037053 2024-11-24T13:47:17,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037045 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037053 2024-11-24T13:47:17,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,059 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037053) roll requested 2024-11-24T13:47:17,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037059 2024-11-24T13:47:17,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037053 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037059 2024-11-24T13:47:17,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,066 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037059) roll requested 2024-11-24T13:47:17,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037066 2024-11-24T13:47:17,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037059 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037066 2024-11-24T13:47:17,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,075 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037066) roll requested 2024-11-24T13:47:17,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037075 2024-11-24T13:47:17,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037066 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037075 2024-11-24T13:47:17,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,083 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037075) roll requested 2024-11-24T13:47:17,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037084 2024-11-24T13:47:17,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037075 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037084 2024-11-24T13:47:17,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,091 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037084) roll requested 2024-11-24T13:47:17,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037091 2024-11-24T13:47:17,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037084 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037091 2024-11-24T13:47:17,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,100 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,101 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=56, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,103 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037091) roll requested 2024-11-24T13:47:17,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037103 2024-11-24T13:47:17,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037091 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037103 2024-11-24T13:47:17,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,110 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=57, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,110 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,111 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037103) roll requested 2024-11-24T13:47:17,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037111 2024-11-24T13:47:17,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037103 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037111 2024-11-24T13:47:17,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037103 is not closed yet, will try archiving it next time 2024-11-24T13:47:17,116 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=58, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,116 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,117 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037111) roll requested 2024-11-24T13:47:17,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037117 2024-11-24T13:47:17,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037111 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037117 2024-11-24T13:47:17,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,162 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=59, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,162 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,162 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037117) roll requested 2024-11-24T13:47:17,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037162 2024-11-24T13:47:17,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037117 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037162 2024-11-24T13:47:17,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,171 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=60, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,171 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,171 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037162) roll requested 2024-11-24T13:47:17,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037171 2024-11-24T13:47:17,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037162 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037171 2024-11-24T13:47:17,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,179 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=61, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,179 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,179 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037171) roll requested 2024-11-24T13:47:17,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037179 2024-11-24T13:47:17,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037171 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037179 2024-11-24T13:47:17,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,185 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=62, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,185 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,185 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037179) roll requested 2024-11-24T13:47:17,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037185 2024-11-24T13:47:17,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037179 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037185 2024-11-24T13:47:17,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,195 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=63, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,195 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,195 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037185) roll requested 2024-11-24T13:47:17,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037195 2024-11-24T13:47:17,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037185 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037195 2024-11-24T13:47:17,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,201 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=64, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,201 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,201 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037195) roll requested 2024-11-24T13:47:17,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037201 2024-11-24T13:47:17,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037195 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037201 2024-11-24T13:47:17,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,211 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=65, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,211 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,211 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037201) roll requested 2024-11-24T13:47:17,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037211 2024-11-24T13:47:17,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037201 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037211 2024-11-24T13:47:17,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,216 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=66, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,217 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,217 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037211) roll requested 2024-11-24T13:47:17,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037217 2024-11-24T13:47:17,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037211 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037217 2024-11-24T13:47:17,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,222 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=67, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,222 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,223 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037217) roll requested 2024-11-24T13:47:17,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037223 2024-11-24T13:47:17,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037217 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037223 2024-11-24T13:47:17,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=68, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,229 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,229 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037223) roll requested 2024-11-24T13:47:17,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037229 2024-11-24T13:47:17,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037223 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037229 2024-11-24T13:47:17,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,235 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=69, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,235 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,235 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037229) roll requested 2024-11-24T13:47:17,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037236 2024-11-24T13:47:17,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037229 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037236 2024-11-24T13:47:17,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,243 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=70, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,243 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,243 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037236) roll requested 2024-11-24T13:47:17,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037243 2024-11-24T13:47:17,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037236 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037243 2024-11-24T13:47:17,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,252 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=71, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,252 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,252 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037243) roll requested 2024-11-24T13:47:17,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037252 2024-11-24T13:47:17,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037243 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037252 2024-11-24T13:47:17,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,260 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=72, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,260 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,260 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037252) roll requested 2024-11-24T13:47:17,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037260 2024-11-24T13:47:17,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037252 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037260 2024-11-24T13:47:17,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,268 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=73, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,268 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,268 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037260) roll requested 2024-11-24T13:47:17,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037268 2024-11-24T13:47:17,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037260 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037268 2024-11-24T13:47:17,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,279 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=74, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,279 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,279 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037268) roll requested 2024-11-24T13:47:17,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037279 2024-11-24T13:47:17,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037268 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037279 2024-11-24T13:47:17,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,289 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=75, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,289 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,289 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037279) roll requested 2024-11-24T13:47:17,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037289 2024-11-24T13:47:17,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037279 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037289 2024-11-24T13:47:17,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,295 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=76, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,295 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,295 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037289) roll requested 2024-11-24T13:47:17,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037295 2024-11-24T13:47:17,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037289 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037295 2024-11-24T13:47:17,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,300 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=77, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,300 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,300 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037295) roll requested 2024-11-24T13:47:17,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037300 2024-11-24T13:47:17,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037295 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037300 2024-11-24T13:47:17,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,306 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=78, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,306 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,306 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037300) roll requested 2024-11-24T13:47:17,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037306 2024-11-24T13:47:17,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037300 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037306 2024-11-24T13:47:17,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,313 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=79, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,313 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,313 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037306) roll requested 2024-11-24T13:47:17,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037313 2024-11-24T13:47:17,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037306 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037313 2024-11-24T13:47:17,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,320 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=80, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,320 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,321 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037313) roll requested 2024-11-24T13:47:17,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037321 2024-11-24T13:47:17,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037313 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037321 2024-11-24T13:47:17,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,328 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=81, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,329 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,329 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037321) roll requested 2024-11-24T13:47:17,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037329 2024-11-24T13:47:17,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037321 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037329 2024-11-24T13:47:17,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,336 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=82, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,336 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,337 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037329) roll requested 2024-11-24T13:47:17,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037337 2024-11-24T13:47:17,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037329 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037337 2024-11-24T13:47:17,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,346 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=83, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,347 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,347 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037337) roll requested 2024-11-24T13:47:17,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037347 2024-11-24T13:47:17,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037337 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037347 2024-11-24T13:47:17,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,355 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=84, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,355 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,355 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037347) roll requested 2024-11-24T13:47:17,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037355 2024-11-24T13:47:17,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037347 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037355 2024-11-24T13:47:17,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,363 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=85, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,363 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,363 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037355) roll requested 2024-11-24T13:47:17,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037363 2024-11-24T13:47:17,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037355 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037363 2024-11-24T13:47:17,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,371 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=86, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,371 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,371 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037363) roll requested 2024-11-24T13:47:17,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037371 2024-11-24T13:47:17,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037363 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037371 2024-11-24T13:47:17,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,377 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=87, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,377 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,377 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037371) roll requested 2024-11-24T13:47:17,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037377 2024-11-24T13:47:17,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037371 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037377 2024-11-24T13:47:17,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,382 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=88, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,382 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,382 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037377) roll requested 2024-11-24T13:47:17,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037382 2024-11-24T13:47:17,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037377 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037382 2024-11-24T13:47:17,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,390 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=89, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,391 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,391 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037382) roll requested 2024-11-24T13:47:17,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037391 2024-11-24T13:47:17,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037382 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037391 2024-11-24T13:47:17,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,396 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=90, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,396 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,397 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037391) roll requested 2024-11-24T13:47:17,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037397 2024-11-24T13:47:17,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037391 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037397 2024-11-24T13:47:17,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,402 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=91, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,402 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,402 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037397) roll requested 2024-11-24T13:47:17,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037402 2024-11-24T13:47:17,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037397 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037402 2024-11-24T13:47:17,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,408 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=92, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,408 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037402) roll requested 2024-11-24T13:47:17,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037408 2024-11-24T13:47:17,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037402 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037408 2024-11-24T13:47:17,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,413 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=93, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,413 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,413 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037408) roll requested 2024-11-24T13:47:17,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037413 2024-11-24T13:47:17,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037408 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037413 2024-11-24T13:47:17,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,418 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=94, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,418 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,418 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037413) roll requested 2024-11-24T13:47:17,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037418 2024-11-24T13:47:17,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037413 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037418 2024-11-24T13:47:17,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,424 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=95, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,424 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,424 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037418) roll requested 2024-11-24T13:47:17,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037424 2024-11-24T13:47:17,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037418 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037424 2024-11-24T13:47:17,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,430 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=96, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,430 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,430 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037424) roll requested 2024-11-24T13:47:17,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037430 2024-11-24T13:47:17,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037424 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037430 2024-11-24T13:47:17,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,435 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=97, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,435 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,435 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037430) roll requested 2024-11-24T13:47:17,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037435 2024-11-24T13:47:17,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037430 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037435 2024-11-24T13:47:17,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,441 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=98, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,441 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,441 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037435) roll requested 2024-11-24T13:47:17,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037441 2024-11-24T13:47:17,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037435 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037441 2024-11-24T13:47:17,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,446 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=99, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,446 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,446 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037441) roll requested 2024-11-24T13:47:17,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037446 2024-11-24T13:47:17,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037441 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037446 2024-11-24T13:47:17,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,452 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=100, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,452 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,452 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037446) roll requested 2024-11-24T13:47:17,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037452 2024-11-24T13:47:17,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037446 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037452 2024-11-24T13:47:17,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,457 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=101, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,457 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,457 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037452) roll requested 2024-11-24T13:47:17,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037458 2024-11-24T13:47:17,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037452 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037458 2024-11-24T13:47:17,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,463 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=102, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,464 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,464 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037458) roll requested 2024-11-24T13:47:17,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037464 2024-11-24T13:47:17,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037458 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037464 2024-11-24T13:47:17,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,469 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=103, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,470 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,470 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037464) roll requested 2024-11-24T13:47:17,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037470 2024-11-24T13:47:17,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037464 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037470 2024-11-24T13:47:17,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,475 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=104, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,475 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,476 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037470) roll requested 2024-11-24T13:47:17,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037476 2024-11-24T13:47:17,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037470 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037476 2024-11-24T13:47:17,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,481 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=105, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,481 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,481 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037476) roll requested 2024-11-24T13:47:17,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037481 2024-11-24T13:47:17,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037476 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037481 2024-11-24T13:47:17,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,486 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=106, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,486 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,486 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037481) roll requested 2024-11-24T13:47:17,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037487 2024-11-24T13:47:17,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037481 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037487 2024-11-24T13:47:17,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,492 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=107, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,492 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,492 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037487) roll requested 2024-11-24T13:47:17,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037492 2024-11-24T13:47:17,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037487 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037492 2024-11-24T13:47:17,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,498 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=108, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,498 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,498 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037492) roll requested 2024-11-24T13:47:17,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037498 2024-11-24T13:47:17,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037492 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037498 2024-11-24T13:47:17,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,503 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=109, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,503 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,503 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037498) roll requested 2024-11-24T13:47:17,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037503 2024-11-24T13:47:17,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037498 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037503 2024-11-24T13:47:17,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,509 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=110, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,509 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,509 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037503) roll requested 2024-11-24T13:47:17,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037509 2024-11-24T13:47:17,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037503 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037509 2024-11-24T13:47:17,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,514 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=111, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,514 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,514 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037509) roll requested 2024-11-24T13:47:17,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037514 2024-11-24T13:47:17,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037509 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037514 2024-11-24T13:47:17,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,520 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=112, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,520 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,520 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037514) roll requested 2024-11-24T13:47:17,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037520 2024-11-24T13:47:17,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037514 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037520 2024-11-24T13:47:17,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,525 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=113, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,525 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,525 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037520) roll requested 2024-11-24T13:47:17,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037525 2024-11-24T13:47:17,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037520 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037525 2024-11-24T13:47:17,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,531 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=114, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,531 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,531 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037525) roll requested 2024-11-24T13:47:17,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037531 2024-11-24T13:47:17,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037525 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037531 2024-11-24T13:47:17,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,538 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=115, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,538 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,538 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037531) roll requested 2024-11-24T13:47:17,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037538 2024-11-24T13:47:17,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037531 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037538 2024-11-24T13:47:17,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,549 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=116, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,550 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,550 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037538) roll requested 2024-11-24T13:47:17,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037550 2024-11-24T13:47:17,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037538 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037550 2024-11-24T13:47:17,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,557 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=117, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,557 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,557 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037550) roll requested 2024-11-24T13:47:17,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037557 2024-11-24T13:47:17,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037550 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037557 2024-11-24T13:47:17,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,566 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=118, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,566 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,566 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037557) roll requested 2024-11-24T13:47:17,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037566 2024-11-24T13:47:17,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037557 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037566 2024-11-24T13:47:17,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,573 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=119, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,574 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,574 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037566) roll requested 2024-11-24T13:47:17,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037574 2024-11-24T13:47:17,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037566 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037574 2024-11-24T13:47:17,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,581 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=120, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,581 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,581 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037574) roll requested 2024-11-24T13:47:17,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037581 2024-11-24T13:47:17,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037574 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037581 2024-11-24T13:47:17,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,586 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=121, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,587 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,587 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037581) roll requested 2024-11-24T13:47:17,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037587 2024-11-24T13:47:17,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037581 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037587 2024-11-24T13:47:17,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,594 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=122, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,594 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,594 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037587) roll requested 2024-11-24T13:47:17,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037594 2024-11-24T13:47:17,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037587 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037594 2024-11-24T13:47:17,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,601 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=123, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,601 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,601 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037594) roll requested 2024-11-24T13:47:17,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037601 2024-11-24T13:47:17,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037594 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037601 2024-11-24T13:47:17,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,607 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=124, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,607 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,607 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037601) roll requested 2024-11-24T13:47:17,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037607 2024-11-24T13:47:17,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037601 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037607 2024-11-24T13:47:17,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,614 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=125, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,614 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,614 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037607) roll requested 2024-11-24T13:47:17,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037615 2024-11-24T13:47:17,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037607 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037615 2024-11-24T13:47:17,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,619 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=126, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,619 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,620 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037615) roll requested 2024-11-24T13:47:17,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037620 2024-11-24T13:47:17,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037615 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037620 2024-11-24T13:47:17,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,625 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=127, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,625 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,625 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037620) roll requested 2024-11-24T13:47:17,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037625 2024-11-24T13:47:17,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037620 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037625 2024-11-24T13:47:17,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,630 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=128, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,630 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,630 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037625) roll requested 2024-11-24T13:47:17,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037630 2024-11-24T13:47:17,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037625 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037630 2024-11-24T13:47:17,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,636 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=129, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,636 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,636 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037630) roll requested 2024-11-24T13:47:17,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037636 2024-11-24T13:47:17,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037630 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037636 2024-11-24T13:47:17,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,641 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=130, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,641 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,641 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037636) roll requested 2024-11-24T13:47:17,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037641 2024-11-24T13:47:17,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037636 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037641 2024-11-24T13:47:17,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,646 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=131, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,646 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,646 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037641) roll requested 2024-11-24T13:47:17,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037646 2024-11-24T13:47:17,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037641 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037646 2024-11-24T13:47:17,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,654 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=132, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,654 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,654 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037646) roll requested 2024-11-24T13:47:17,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037654 2024-11-24T13:47:17,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037646 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037654 2024-11-24T13:47:17,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,660 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=133, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,660 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,660 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037654) roll requested 2024-11-24T13:47:17,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037660 2024-11-24T13:47:17,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037654 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037660 2024-11-24T13:47:17,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,666 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=134, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,666 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,667 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037660) roll requested 2024-11-24T13:47:17,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037667 2024-11-24T13:47:17,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037660 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037667 2024-11-24T13:47:17,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,675 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=135, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,675 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,675 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037667) roll requested 2024-11-24T13:47:17,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037675 2024-11-24T13:47:17,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037667 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037675 2024-11-24T13:47:17,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,684 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=136, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,684 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,684 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037675) roll requested 2024-11-24T13:47:17,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037684 2024-11-24T13:47:17,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037675 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037684 2024-11-24T13:47:17,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,695 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=137, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,695 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,695 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037684) roll requested 2024-11-24T13:47:17,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037695 2024-11-24T13:47:17,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037684 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037695 2024-11-24T13:47:17,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,701 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=138, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,701 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,702 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037695) roll requested 2024-11-24T13:47:17,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037702 2024-11-24T13:47:17,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037695 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037702 2024-11-24T13:47:17,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,708 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=139, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,708 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037702) roll requested 2024-11-24T13:47:17,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037708 2024-11-24T13:47:17,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037702 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037708 2024-11-24T13:47:17,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,716 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=140, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,716 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,716 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037708) roll requested 2024-11-24T13:47:17,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037716 2024-11-24T13:47:17,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037708 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037716 2024-11-24T13:47:17,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,721 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,721 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=141, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,721 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037716) roll requested 2024-11-24T13:47:17,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037721 2024-11-24T13:47:17,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037716 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037721 2024-11-24T13:47:17,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,726 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=142, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,726 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,726 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037721) roll requested 2024-11-24T13:47:17,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037726 2024-11-24T13:47:17,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037721 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037726 2024-11-24T13:47:17,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,732 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=143, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,732 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,732 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037726) roll requested 2024-11-24T13:47:17,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037732 2024-11-24T13:47:17,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037726 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037732 2024-11-24T13:47:17,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,737 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=144, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,737 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,737 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037732) roll requested 2024-11-24T13:47:17,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037737 2024-11-24T13:47:17,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037732 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037737 2024-11-24T13:47:17,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,742 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,743 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=145, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,743 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037737) roll requested 2024-11-24T13:47:17,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037743 2024-11-24T13:47:17,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037737 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037743 2024-11-24T13:47:17,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037737 is not closed yet, will try archiving it next time 2024-11-24T13:47:17,748 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=146, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,748 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,748 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037743) roll requested 2024-11-24T13:47:17,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037748 2024-11-24T13:47:17,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037743 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037748 2024-11-24T13:47:17,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,753 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=147, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,754 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,754 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037748) roll requested 2024-11-24T13:47:17,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037754 2024-11-24T13:47:17,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037748 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037754 2024-11-24T13:47:17,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,759 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=148, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,759 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,759 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037754) roll requested 2024-11-24T13:47:17,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037759 2024-11-24T13:47:17,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037754 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037759 2024-11-24T13:47:17,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,764 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=149, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,764 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,764 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037759) roll requested 2024-11-24T13:47:17,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037764 2024-11-24T13:47:17,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037759 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037764 2024-11-24T13:47:17,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,769 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=150, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,769 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,769 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037764) roll requested 2024-11-24T13:47:17,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037769 2024-11-24T13:47:17,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037764 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037769 2024-11-24T13:47:17,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,775 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=151, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,775 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,775 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037769) roll requested 2024-11-24T13:47:17,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037775 2024-11-24T13:47:17,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037769 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037775 2024-11-24T13:47:17,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,785 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=152, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,786 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,786 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037775) roll requested 2024-11-24T13:47:17,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037786 2024-11-24T13:47:17,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037775 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037786 2024-11-24T13:47:17,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,793 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=153, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,793 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,794 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037786) roll requested 2024-11-24T13:47:17,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037794 2024-11-24T13:47:17,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037786 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037794 2024-11-24T13:47:17,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,799 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=154, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,799 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,799 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037794) roll requested 2024-11-24T13:47:17,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037799 2024-11-24T13:47:17,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037794 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037799 2024-11-24T13:47:17,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,807 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=155, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,807 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,807 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037799) roll requested 2024-11-24T13:47:17,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037807 2024-11-24T13:47:17,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037799 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037807 2024-11-24T13:47:17,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,815 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,815 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=156, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,815 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037807) roll requested 2024-11-24T13:47:17,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037815 2024-11-24T13:47:17,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037807 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037815 2024-11-24T13:47:17,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,827 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=157, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,827 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,827 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037815) roll requested 2024-11-24T13:47:17,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037827 2024-11-24T13:47:17,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037815 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037827 2024-11-24T13:47:17,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,835 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=158, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,835 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,835 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037827) roll requested 2024-11-24T13:47:17,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037835 2024-11-24T13:47:17,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037827 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037835 2024-11-24T13:47:17,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,840 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=159, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,840 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,841 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037835) roll requested 2024-11-24T13:47:17,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037841 2024-11-24T13:47:17,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037835 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037841 2024-11-24T13:47:17,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,866 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=160, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,866 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,867 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037841) roll requested 2024-11-24T13:47:17,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037867 2024-11-24T13:47:17,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037841 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037867 2024-11-24T13:47:17,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,879 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,879 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=161, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,879 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037867) roll requested 2024-11-24T13:47:17,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037879 2024-11-24T13:47:17,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037867 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037879 2024-11-24T13:47:17,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,891 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=162, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,891 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,891 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037879) roll requested 2024-11-24T13:47:17,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037891 2024-11-24T13:47:17,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037879 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037891 2024-11-24T13:47:17,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,899 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=163, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,899 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,899 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037891) roll requested 2024-11-24T13:47:17,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037899 2024-11-24T13:47:17,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037891 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037899 2024-11-24T13:47:17,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,907 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=164, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,907 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,907 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037899) roll requested 2024-11-24T13:47:17,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037907 2024-11-24T13:47:17,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037899 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037907 2024-11-24T13:47:17,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,918 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=165, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,918 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,918 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037907) roll requested 2024-11-24T13:47:17,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037919 2024-11-24T13:47:17,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037907 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037919 2024-11-24T13:47:17,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,927 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=166, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,927 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,927 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037919) roll requested 2024-11-24T13:47:17,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037927 2024-11-24T13:47:17,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037919 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037927 2024-11-24T13:47:17,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,935 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=167, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,935 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,935 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037927) roll requested 2024-11-24T13:47:17,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037935 2024-11-24T13:47:17,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037927 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037935 2024-11-24T13:47:17,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,943 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=168, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,943 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,943 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037935) roll requested 2024-11-24T13:47:17,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037943 2024-11-24T13:47:17,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037935 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037943 2024-11-24T13:47:17,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,951 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=169, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,951 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,951 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037943) roll requested 2024-11-24T13:47:17,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037951 2024-11-24T13:47:17,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037943 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037951 2024-11-24T13:47:17,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,958 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=170, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,958 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,958 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037951) roll requested 2024-11-24T13:47:17,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037959 2024-11-24T13:47:17,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037951 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037959 2024-11-24T13:47:17,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,965 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=171, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,965 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,965 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037959) roll requested 2024-11-24T13:47:17,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037965 2024-11-24T13:47:17,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037959 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037965 2024-11-24T13:47:17,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,971 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=172, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,971 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,972 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037965) roll requested 2024-11-24T13:47:17,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037972 2024-11-24T13:47:17,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037965 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037972 2024-11-24T13:47:17,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,987 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=173, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,987 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,987 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037972) roll requested 2024-11-24T13:47:17,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037987 2024-11-24T13:47:17,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037972 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037987 2024-11-24T13:47:17,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,993 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=174, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,993 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,993 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037987) roll requested 2024-11-24T13:47:17,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037993 2024-11-24T13:47:17,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:17,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037987 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037993 2024-11-24T13:47:17,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:17,998 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,998 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=175, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:17,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:17,998 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:17,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037993) roll requested 2024-11-24T13:47:17,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456037999 2024-11-24T13:47:18,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037993 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037999 2024-11-24T13:47:18,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,014 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,014 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=176, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,015 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456037999) roll requested 2024-11-24T13:47:18,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038015 2024-11-24T13:47:18,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456037999 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038015 2024-11-24T13:47:18,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,020 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,020 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=177, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,020 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038015) roll requested 2024-11-24T13:47:18,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038020 2024-11-24T13:47:18,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038015 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038020 2024-11-24T13:47:18,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,025 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,025 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=178, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,025 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038020) roll requested 2024-11-24T13:47:18,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038025 2024-11-24T13:47:18,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038020 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038025 2024-11-24T13:47:18,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,030 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=179, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,030 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,031 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038025) roll requested 2024-11-24T13:47:18,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038031 2024-11-24T13:47:18,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038025 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038031 2024-11-24T13:47:18,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,037 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=180, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,037 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,037 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038031) roll requested 2024-11-24T13:47:18,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038038 2024-11-24T13:47:18,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038031 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038038 2024-11-24T13:47:18,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,046 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=181, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,046 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,046 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038038) roll requested 2024-11-24T13:47:18,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038046 2024-11-24T13:47:18,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038038 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038046 2024-11-24T13:47:18,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,051 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,051 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=182, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,051 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038046) roll requested 2024-11-24T13:47:18,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038051 2024-11-24T13:47:18,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038046 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038051 2024-11-24T13:47:18,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,057 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,057 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=183, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,057 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038051) roll requested 2024-11-24T13:47:18,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038057 2024-11-24T13:47:18,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038051 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038057 2024-11-24T13:47:18,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,064 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=184, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,064 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,064 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038057) roll requested 2024-11-24T13:47:18,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038064 2024-11-24T13:47:18,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038057 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038064 2024-11-24T13:47:18,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,072 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,072 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=185, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,072 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038064) roll requested 2024-11-24T13:47:18,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038072 2024-11-24T13:47:18,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038064 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038072 2024-11-24T13:47:18,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,081 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=186, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,081 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,082 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038072) roll requested 2024-11-24T13:47:18,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038082 2024-11-24T13:47:18,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038072 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038082 2024-11-24T13:47:18,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,090 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=187, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,090 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,090 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038082) roll requested 2024-11-24T13:47:18,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038090 2024-11-24T13:47:18,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038082 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038090 2024-11-24T13:47:18,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,097 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=188, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,097 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,097 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038090) roll requested 2024-11-24T13:47:18,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038097 2024-11-24T13:47:18,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038090 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038097 2024-11-24T13:47:18,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=189, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,104 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038097) roll requested 2024-11-24T13:47:18,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038104 2024-11-24T13:47:18,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038097 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038104 2024-11-24T13:47:18,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,112 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=190, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,112 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,112 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038104) roll requested 2024-11-24T13:47:18,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038112 2024-11-24T13:47:18,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038104 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038112 2024-11-24T13:47:18,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,121 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=191, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,121 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,121 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038112) roll requested 2024-11-24T13:47:18,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038121 2024-11-24T13:47:18,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038112 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038121 2024-11-24T13:47:18,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,130 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=192, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,130 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,130 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038121) roll requested 2024-11-24T13:47:18,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038130 2024-11-24T13:47:18,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038121 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038130 2024-11-24T13:47:18,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,139 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,139 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=193, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,139 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038130) roll requested 2024-11-24T13:47:18,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038139 2024-11-24T13:47:18,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038130 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038139 2024-11-24T13:47:18,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,145 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=194, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,145 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,145 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038139) roll requested 2024-11-24T13:47:18,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038145 2024-11-24T13:47:18,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038139 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038145 2024-11-24T13:47:18,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,150 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=195, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,150 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,150 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038145) roll requested 2024-11-24T13:47:18,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038150 2024-11-24T13:47:18,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038145 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038150 2024-11-24T13:47:18,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,155 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,155 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=196, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,156 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038150) roll requested 2024-11-24T13:47:18,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038156 2024-11-24T13:47:18,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038150 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038156 2024-11-24T13:47:18,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,161 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=197, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,161 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,161 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038156) roll requested 2024-11-24T13:47:18,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038161 2024-11-24T13:47:18,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038156 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038161 2024-11-24T13:47:18,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,170 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=198, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,170 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,170 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038161) roll requested 2024-11-24T13:47:18,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038171 2024-11-24T13:47:18,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038161 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038171 2024-11-24T13:47:18,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,178 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=199, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,178 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,178 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038171) roll requested 2024-11-24T13:47:18,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038179 2024-11-24T13:47:18,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038171 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038179 2024-11-24T13:47:18,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,189 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=200, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,189 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,189 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038179) roll requested 2024-11-24T13:47:18,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038189 2024-11-24T13:47:18,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038179 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038189 2024-11-24T13:47:18,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,198 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=201, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,198 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,198 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038189) roll requested 2024-11-24T13:47:18,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038198 2024-11-24T13:47:18,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038189 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038198 2024-11-24T13:47:18,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,207 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=202, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,207 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,207 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038198) roll requested 2024-11-24T13:47:18,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038207 2024-11-24T13:47:18,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038198 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038207 2024-11-24T13:47:18,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,217 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=203, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,217 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,218 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038207) roll requested 2024-11-24T13:47:18,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038218 2024-11-24T13:47:18,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038207 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038218 2024-11-24T13:47:18,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,225 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=204, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,225 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,225 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038218) roll requested 2024-11-24T13:47:18,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038226 2024-11-24T13:47:18,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038218 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038226 2024-11-24T13:47:18,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,234 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=205, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,234 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,234 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038226) roll requested 2024-11-24T13:47:18,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038234 2024-11-24T13:47:18,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038226 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038234 2024-11-24T13:47:18,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,248 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=206, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,248 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,248 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038234) roll requested 2024-11-24T13:47:18,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038248 2024-11-24T13:47:18,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038234 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038248 2024-11-24T13:47:18,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,254 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=207, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,254 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,255 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038248) roll requested 2024-11-24T13:47:18,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038255 2024-11-24T13:47:18,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038248 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038255 2024-11-24T13:47:18,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,268 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=208, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,268 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,269 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038255) roll requested 2024-11-24T13:47:18,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038269 2024-11-24T13:47:18,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038255 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038269 2024-11-24T13:47:18,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,276 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=209, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,276 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,276 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038269) roll requested 2024-11-24T13:47:18,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038276 2024-11-24T13:47:18,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038269 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038276 2024-11-24T13:47:18,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,282 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=210, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,282 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,282 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038276) roll requested 2024-11-24T13:47:18,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038282 2024-11-24T13:47:18,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038276 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038282 2024-11-24T13:47:18,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,288 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=211, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,288 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,289 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038282) roll requested 2024-11-24T13:47:18,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038289 2024-11-24T13:47:18,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038282 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038289 2024-11-24T13:47:18,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,297 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=212, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,297 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,297 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038289) roll requested 2024-11-24T13:47:18,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038297 2024-11-24T13:47:18,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038289 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038297 2024-11-24T13:47:18,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,303 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=213, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,303 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,303 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038297) roll requested 2024-11-24T13:47:18,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038303 2024-11-24T13:47:18,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038297 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038303 2024-11-24T13:47:18,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,309 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=214, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,309 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,309 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038303) roll requested 2024-11-24T13:47:18,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038310 2024-11-24T13:47:18,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038303 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038310 2024-11-24T13:47:18,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,317 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=215, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,317 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,317 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038310) roll requested 2024-11-24T13:47:18,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038317 2024-11-24T13:47:18,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038310 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038317 2024-11-24T13:47:18,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,322 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,322 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=216, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,322 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038317) roll requested 2024-11-24T13:47:18,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038323 2024-11-24T13:47:18,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038317 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038323 2024-11-24T13:47:18,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,329 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,329 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=217, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,329 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038323) roll requested 2024-11-24T13:47:18,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038329 2024-11-24T13:47:18,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038323 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038329 2024-11-24T13:47:18,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,335 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,335 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=218, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,336 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038329) roll requested 2024-11-24T13:47:18,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038336 2024-11-24T13:47:18,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038329 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038336 2024-11-24T13:47:18,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,343 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=219, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,343 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,343 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038336) roll requested 2024-11-24T13:47:18,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038343 2024-11-24T13:47:18,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038336 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038343 2024-11-24T13:47:18,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,351 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=220, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,351 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,351 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038343) roll requested 2024-11-24T13:47:18,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038351 2024-11-24T13:47:18,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038343 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038351 2024-11-24T13:47:18,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,362 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,362 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=221, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,362 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038351) roll requested 2024-11-24T13:47:18,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038362 2024-11-24T13:47:18,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038351 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038362 2024-11-24T13:47:18,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038351 is not closed yet, will try archiving it next time 2024-11-24T13:47:18,371 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,371 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=222, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,371 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038362) roll requested 2024-11-24T13:47:18,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038371 2024-11-24T13:47:18,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038362 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038371 2024-11-24T13:47:18,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,379 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=223, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,379 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,379 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038371) roll requested 2024-11-24T13:47:18,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038379 2024-11-24T13:47:18,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038371 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038379 2024-11-24T13:47:18,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,385 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=224, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,385 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,385 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038379) roll requested 2024-11-24T13:47:18,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038385 2024-11-24T13:47:18,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038379 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038385 2024-11-24T13:47:18,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,394 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=225, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,394 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,394 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038385) roll requested 2024-11-24T13:47:18,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038394 2024-11-24T13:47:18,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038385 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038394 2024-11-24T13:47:18,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,403 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=226, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,403 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,403 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038394) roll requested 2024-11-24T13:47:18,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038403 2024-11-24T13:47:18,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038394 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038403 2024-11-24T13:47:18,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038394 is not closed yet, will try archiving it next time 2024-11-24T13:47:18,419 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=227, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,420 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,420 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038403) roll requested 2024-11-24T13:47:18,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038420 2024-11-24T13:47:18,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038403 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038420 2024-11-24T13:47:18,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,426 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=228, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,426 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,427 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038420) roll requested 2024-11-24T13:47:18,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038427 2024-11-24T13:47:18,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038420 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038427 2024-11-24T13:47:18,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,432 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=229, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,432 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,432 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038427) roll requested 2024-11-24T13:47:18,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038432 2024-11-24T13:47:18,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038427 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038432 2024-11-24T13:47:18,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,438 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,438 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=230, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,438 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038432) roll requested 2024-11-24T13:47:18,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038438 2024-11-24T13:47:18,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038432 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038438 2024-11-24T13:47:18,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,450 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,450 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=231, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,450 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038438) roll requested 2024-11-24T13:47:18,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038450 2024-11-24T13:47:18,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038438 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038450 2024-11-24T13:47:18,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,457 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,457 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=232, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,457 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038450) roll requested 2024-11-24T13:47:18,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038457 2024-11-24T13:47:18,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038450 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038457 2024-11-24T13:47:18,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,463 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=233, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,463 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,464 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038457) roll requested 2024-11-24T13:47:18,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038464 2024-11-24T13:47:18,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038457 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038464 2024-11-24T13:47:18,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,469 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=234, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,469 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,469 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038464) roll requested 2024-11-24T13:47:18,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038469 2024-11-24T13:47:18,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038464 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038469 2024-11-24T13:47:18,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,474 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=235, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,474 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,475 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038469) roll requested 2024-11-24T13:47:18,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038475 2024-11-24T13:47:18,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038469 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038475 2024-11-24T13:47:18,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,481 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,481 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=236, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,481 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038475) roll requested 2024-11-24T13:47:18,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038481 2024-11-24T13:47:18,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038475 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038481 2024-11-24T13:47:18,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,489 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,489 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=237, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,489 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038481) roll requested 2024-11-24T13:47:18,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038489 2024-11-24T13:47:18,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038481 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038489 2024-11-24T13:47:18,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,498 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,498 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=238, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,498 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038489) roll requested 2024-11-24T13:47:18,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038498 2024-11-24T13:47:18,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038489 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038498 2024-11-24T13:47:18,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,507 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=239, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,507 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,507 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038498) roll requested 2024-11-24T13:47:18,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038507 2024-11-24T13:47:18,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038498 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038507 2024-11-24T13:47:18,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,516 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=240, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,516 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,516 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038507) roll requested 2024-11-24T13:47:18,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038516 2024-11-24T13:47:18,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038507 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038516 2024-11-24T13:47:18,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,524 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=241, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,524 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,524 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038516) roll requested 2024-11-24T13:47:18,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038524 2024-11-24T13:47:18,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038516 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038524 2024-11-24T13:47:18,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,531 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=242, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,531 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,531 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038524) roll requested 2024-11-24T13:47:18,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038531 2024-11-24T13:47:18,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038524 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038531 2024-11-24T13:47:18,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,538 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,538 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=243, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,538 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038531) roll requested 2024-11-24T13:47:18,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038539 2024-11-24T13:47:18,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038531 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038539 2024-11-24T13:47:18,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,546 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=244, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,546 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,547 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038539) roll requested 2024-11-24T13:47:18,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038547 2024-11-24T13:47:18,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038539 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038547 2024-11-24T13:47:18,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,555 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,555 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=245, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,556 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038547) roll requested 2024-11-24T13:47:18,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038556 2024-11-24T13:47:18,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038547 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038556 2024-11-24T13:47:18,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,563 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,563 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=246, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,563 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038556) roll requested 2024-11-24T13:47:18,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038564 2024-11-24T13:47:18,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038556 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038564 2024-11-24T13:47:18,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,572 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,572 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=247, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,572 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038564) roll requested 2024-11-24T13:47:18,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038572 2024-11-24T13:47:18,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038564 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038572 2024-11-24T13:47:18,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,581 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,581 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=248, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,581 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038572) roll requested 2024-11-24T13:47:18,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038581 2024-11-24T13:47:18,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038572 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038581 2024-11-24T13:47:18,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,588 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=249, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,588 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,589 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038581) roll requested 2024-11-24T13:47:18,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038589 2024-11-24T13:47:18,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038581 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038589 2024-11-24T13:47:18,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,596 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,596 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=250, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,597 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038589) roll requested 2024-11-24T13:47:18,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038597 2024-11-24T13:47:18,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038589 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038597 2024-11-24T13:47:18,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,608 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=251, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,608 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,609 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038597) roll requested 2024-11-24T13:47:18,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038609 2024-11-24T13:47:18,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038597 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038609 2024-11-24T13:47:18,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,614 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,614 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=252, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,614 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038609) roll requested 2024-11-24T13:47:18,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038615 2024-11-24T13:47:18,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038609 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038615 2024-11-24T13:47:18,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,624 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=253, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,624 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,624 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038615) roll requested 2024-11-24T13:47:18,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038624 2024-11-24T13:47:18,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038615 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038624 2024-11-24T13:47:18,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,630 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=254, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,630 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,630 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038624) roll requested 2024-11-24T13:47:18,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038630 2024-11-24T13:47:18,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038624 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038630 2024-11-24T13:47:18,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038624 is not closed yet, will try archiving it next time 2024-11-24T13:47:18,636 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,636 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=255, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,636 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038630) roll requested 2024-11-24T13:47:18,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038636 2024-11-24T13:47:18,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038630 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038636 2024-11-24T13:47:18,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,651 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=256, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,651 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,651 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038636) roll requested 2024-11-24T13:47:18,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038651 2024-11-24T13:47:18,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038636 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038651 2024-11-24T13:47:18,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,658 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=257, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,658 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,658 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038651) roll requested 2024-11-24T13:47:18,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038658 2024-11-24T13:47:18,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038651 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038658 2024-11-24T13:47:18,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,664 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,664 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=258, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,664 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038658) roll requested 2024-11-24T13:47:18,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038664 2024-11-24T13:47:18,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038658 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038664 2024-11-24T13:47:18,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,670 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,671 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=259, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,671 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038664) roll requested 2024-11-24T13:47:18,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038671 2024-11-24T13:47:18,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038664 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038671 2024-11-24T13:47:18,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,678 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,678 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=260, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,678 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038671) roll requested 2024-11-24T13:47:18,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038678 2024-11-24T13:47:18,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038671 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038678 2024-11-24T13:47:18,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,685 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,685 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=261, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,686 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038678) roll requested 2024-11-24T13:47:18,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038686 2024-11-24T13:47:18,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038678 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038686 2024-11-24T13:47:18,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,693 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,693 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=262, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,694 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038686) roll requested 2024-11-24T13:47:18,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038694 2024-11-24T13:47:18,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038686 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038694 2024-11-24T13:47:18,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,700 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,701 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=263, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,701 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038694) roll requested 2024-11-24T13:47:18,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038701 2024-11-24T13:47:18,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038694 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038701 2024-11-24T13:47:18,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,706 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,706 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=264, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,706 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038701) roll requested 2024-11-24T13:47:18,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038707 2024-11-24T13:47:18,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038707 2024-11-24T13:47:18,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,713 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,713 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=265, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,713 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038707) roll requested 2024-11-24T13:47:18,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038713 2024-11-24T13:47:18,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038707 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038713 2024-11-24T13:47:18,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038707 is not closed yet, will try archiving it next time 2024-11-24T13:47:18,720 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,720 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=266, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,720 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038713) roll requested 2024-11-24T13:47:18,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038721 2024-11-24T13:47:18,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038713 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038721 2024-11-24T13:47:18,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,729 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,729 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=267, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,729 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038721) roll requested 2024-11-24T13:47:18,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038729 2024-11-24T13:47:18,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038721 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038729 2024-11-24T13:47:18,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,737 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=268, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,737 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,737 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038729) roll requested 2024-11-24T13:47:18,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038737 2024-11-24T13:47:18,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038729 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038737 2024-11-24T13:47:18,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,746 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,746 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=269, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,746 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038737) roll requested 2024-11-24T13:47:18,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038746 2024-11-24T13:47:18,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038737 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038746 2024-11-24T13:47:18,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,755 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,755 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=270, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,756 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038746) roll requested 2024-11-24T13:47:18,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038756 2024-11-24T13:47:18,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038746 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038756 2024-11-24T13:47:18,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,763 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,764 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=271, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,764 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038756) roll requested 2024-11-24T13:47:18,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038764 2024-11-24T13:47:18,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038756 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038764 2024-11-24T13:47:18,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,773 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,773 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=272, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,774 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038764) roll requested 2024-11-24T13:47:18,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038774 2024-11-24T13:47:18,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038764 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038774 2024-11-24T13:47:18,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,782 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,782 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=273, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,782 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038774) roll requested 2024-11-24T13:47:18,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038782 2024-11-24T13:47:18,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038774 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038782 2024-11-24T13:47:18,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,788 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,788 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=274, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,788 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038782) roll requested 2024-11-24T13:47:18,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038788 2024-11-24T13:47:18,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038782 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038788 2024-11-24T13:47:18,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,794 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,794 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=275, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,794 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038788) roll requested 2024-11-24T13:47:18,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038794 2024-11-24T13:47:18,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038788 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038794 2024-11-24T13:47:18,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,799 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,799 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=276, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,799 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038794) roll requested 2024-11-24T13:47:18,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038799 2024-11-24T13:47:18,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038794 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038799 2024-11-24T13:47:18,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,804 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,804 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=277, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,804 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038799) roll requested 2024-11-24T13:47:18,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038804 2024-11-24T13:47:18,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038799 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038804 2024-11-24T13:47:18,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,809 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,809 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=278, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,810 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038804) roll requested 2024-11-24T13:47:18,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038810 2024-11-24T13:47:18,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038804 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038810 2024-11-24T13:47:18,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,816 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,816 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=279, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,816 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038810) roll requested 2024-11-24T13:47:18,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038816 2024-11-24T13:47:18,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038810 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038816 2024-11-24T13:47:18,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,823 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,823 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=280, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,823 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038816) roll requested 2024-11-24T13:47:18,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038823 2024-11-24T13:47:18,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038816 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038823 2024-11-24T13:47:18,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,830 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,830 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=281, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,830 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038823) roll requested 2024-11-24T13:47:18,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038831 2024-11-24T13:47:18,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038823 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038831 2024-11-24T13:47:18,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,839 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,839 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=282, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,839 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038831) roll requested 2024-11-24T13:47:18,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038839 2024-11-24T13:47:18,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038831 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038839 2024-11-24T13:47:18,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,847 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,847 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=283, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,847 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038839) roll requested 2024-11-24T13:47:18,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038847 2024-11-24T13:47:18,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038839 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038847 2024-11-24T13:47:18,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,853 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,854 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=284, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,854 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038847) roll requested 2024-11-24T13:47:18,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038854 2024-11-24T13:47:18,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038847 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038854 2024-11-24T13:47:18,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,861 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,862 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=285, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,862 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038854) roll requested 2024-11-24T13:47:18,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038862 2024-11-24T13:47:18,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038854 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038862 2024-11-24T13:47:18,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,871 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=286, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,871 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,871 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038862) roll requested 2024-11-24T13:47:18,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038871 2024-11-24T13:47:18,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038862 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038871 2024-11-24T13:47:18,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,880 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,881 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=287, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,881 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038871) roll requested 2024-11-24T13:47:18,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038881 2024-11-24T13:47:18,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038871 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038881 2024-11-24T13:47:18,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,888 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,888 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=288, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,888 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038881) roll requested 2024-11-24T13:47:18,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038889 2024-11-24T13:47:18,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038881 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038889 2024-11-24T13:47:18,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,894 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,894 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=289, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,894 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038889) roll requested 2024-11-24T13:47:18,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038894 2024-11-24T13:47:18,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038889 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038894 2024-11-24T13:47:18,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,899 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,899 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=290, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,899 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038894) roll requested 2024-11-24T13:47:18,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038899 2024-11-24T13:47:18,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038894 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038899 2024-11-24T13:47:18,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,906 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,906 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=291, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,906 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038899) roll requested 2024-11-24T13:47:18,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038907 2024-11-24T13:47:18,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038899 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038907 2024-11-24T13:47:18,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,913 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,913 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=292, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,914 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038907) roll requested 2024-11-24T13:47:18,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038914 2024-11-24T13:47:18,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038907 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038914 2024-11-24T13:47:18,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,926 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=293, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,927 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,929 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038914) roll requested 2024-11-24T13:47:18,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038929 2024-11-24T13:47:18,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038914 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038929 2024-11-24T13:47:18,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,935 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,935 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=294, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,935 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038929) roll requested 2024-11-24T13:47:18,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038935 2024-11-24T13:47:18,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038929 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038935 2024-11-24T13:47:18,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,948 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=295, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,948 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,948 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038935) roll requested 2024-11-24T13:47:18,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038948 2024-11-24T13:47:18,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038935 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038948 2024-11-24T13:47:18,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,955 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=296, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,955 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,955 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038948) roll requested 2024-11-24T13:47:18,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038955 2024-11-24T13:47:18,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038948 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038955 2024-11-24T13:47:18,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,963 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=297, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,963 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,964 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038955) roll requested 2024-11-24T13:47:18,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038964 2024-11-24T13:47:18,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038955 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038964 2024-11-24T13:47:18,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,970 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,970 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=298, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,970 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038964) roll requested 2024-11-24T13:47:18,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038970 2024-11-24T13:47:18,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038964 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038970 2024-11-24T13:47:18,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,977 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=299, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,977 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,977 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038970) roll requested 2024-11-24T13:47:18,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038977 2024-11-24T13:47:18,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038970 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038977 2024-11-24T13:47:18,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:18,994 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,994 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=300, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:18,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:18,994 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:18,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038977) roll requested 2024-11-24T13:47:18,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456038994 2024-11-24T13:47:18,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:18,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038977 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038994 2024-11-24T13:47:19,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,003 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,003 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=301, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,003 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456038994) roll requested 2024-11-24T13:47:19,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039003 2024-11-24T13:47:19,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456038994 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039003 2024-11-24T13:47:19,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,011 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,011 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=302, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,012 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039003) roll requested 2024-11-24T13:47:19,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039012 2024-11-24T13:47:19,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039003 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039012 2024-11-24T13:47:19,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,020 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=303, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,020 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,020 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039012) roll requested 2024-11-24T13:47:19,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039020 2024-11-24T13:47:19,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039012 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039020 2024-11-24T13:47:19,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,028 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,028 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=304, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,028 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039020) roll requested 2024-11-24T13:47:19,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039029 2024-11-24T13:47:19,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039020 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039029 2024-11-24T13:47:19,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,037 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,037 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=305, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,037 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039029) roll requested 2024-11-24T13:47:19,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039037 2024-11-24T13:47:19,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039029 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039037 2024-11-24T13:47:19,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,043 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,043 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=306, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,043 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039037) roll requested 2024-11-24T13:47:19,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039043 2024-11-24T13:47:19,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039037 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039043 2024-11-24T13:47:19,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,051 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,051 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=307, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,051 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039043) roll requested 2024-11-24T13:47:19,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039051 2024-11-24T13:47:19,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039043 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039051 2024-11-24T13:47:19,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,058 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,059 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=308, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,059 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039051) roll requested 2024-11-24T13:47:19,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039059 2024-11-24T13:47:19,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039051 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039059 2024-11-24T13:47:19,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,067 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=309, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,067 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,067 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039059) roll requested 2024-11-24T13:47:19,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039067 2024-11-24T13:47:19,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039059 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039067 2024-11-24T13:47:19,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,074 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,075 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=310, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,075 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039067) roll requested 2024-11-24T13:47:19,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039075 2024-11-24T13:47:19,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039067 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039075 2024-11-24T13:47:19,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,081 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,081 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=311, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,081 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039075) roll requested 2024-11-24T13:47:19,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039081 2024-11-24T13:47:19,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039075 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039081 2024-11-24T13:47:19,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,086 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,086 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=312, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,087 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039081) roll requested 2024-11-24T13:47:19,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039087 2024-11-24T13:47:19,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039081 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039087 2024-11-24T13:47:19,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,092 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=313, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,092 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,092 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039087) roll requested 2024-11-24T13:47:19,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039092 2024-11-24T13:47:19,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039087 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039092 2024-11-24T13:47:19,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,097 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=314, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,097 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,097 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039092) roll requested 2024-11-24T13:47:19,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039097 2024-11-24T13:47:19,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039092 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039097 2024-11-24T13:47:19,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,103 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,103 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=315, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,103 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039097) roll requested 2024-11-24T13:47:19,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039103 2024-11-24T13:47:19,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039097 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039103 2024-11-24T13:47:19,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,109 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=316, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,109 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,109 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039103) roll requested 2024-11-24T13:47:19,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039110 2024-11-24T13:47:19,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039103 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039110 2024-11-24T13:47:19,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,115 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,115 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=317, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,115 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039110) roll requested 2024-11-24T13:47:19,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039115 2024-11-24T13:47:19,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039110 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039115 2024-11-24T13:47:19,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,121 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=318, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,121 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,122 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039115) roll requested 2024-11-24T13:47:19,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039122 2024-11-24T13:47:19,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039115 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039122 2024-11-24T13:47:19,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,127 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=319, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,127 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,127 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039122) roll requested 2024-11-24T13:47:19,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039127 2024-11-24T13:47:19,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039122 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039127 2024-11-24T13:47:19,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,133 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,133 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=320, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,134 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039127) roll requested 2024-11-24T13:47:19,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039134 2024-11-24T13:47:19,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039127 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039134 2024-11-24T13:47:19,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,139 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,139 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=321, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,140 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039134) roll requested 2024-11-24T13:47:19,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039140 2024-11-24T13:47:19,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039134 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039140 2024-11-24T13:47:19,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,145 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,145 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=322, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,145 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039140) roll requested 2024-11-24T13:47:19,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039145 2024-11-24T13:47:19,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039140 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039145 2024-11-24T13:47:19,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,150 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=323, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,150 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,150 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039145) roll requested 2024-11-24T13:47:19,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039150 2024-11-24T13:47:19,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039145 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039150 2024-11-24T13:47:19,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,155 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,155 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=324, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,156 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039150) roll requested 2024-11-24T13:47:19,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039156 2024-11-24T13:47:19,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039150 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039156 2024-11-24T13:47:19,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,161 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,161 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=325, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,161 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039156) roll requested 2024-11-24T13:47:19,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039161 2024-11-24T13:47:19,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039156 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039161 2024-11-24T13:47:19,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,168 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,168 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=326, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,168 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039161) roll requested 2024-11-24T13:47:19,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039168 2024-11-24T13:47:19,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039161 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039168 2024-11-24T13:47:19,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,176 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,176 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=327, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,176 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039168) roll requested 2024-11-24T13:47:19,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039176 2024-11-24T13:47:19,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039168 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039176 2024-11-24T13:47:19,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,184 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,184 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=328, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,184 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039176) roll requested 2024-11-24T13:47:19,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039185 2024-11-24T13:47:19,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039176 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039185 2024-11-24T13:47:19,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,193 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,193 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=329, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,194 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039185) roll requested 2024-11-24T13:47:19,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039194 2024-11-24T13:47:19,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039185 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039194 2024-11-24T13:47:19,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,204 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=330, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,204 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,204 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039194) roll requested 2024-11-24T13:47:19,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039204 2024-11-24T13:47:19,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039194 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039204 2024-11-24T13:47:19,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,210 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,210 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=331, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,210 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039204) roll requested 2024-11-24T13:47:19,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039211 2024-11-24T13:47:19,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039204 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039211 2024-11-24T13:47:19,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,216 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,217 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=332, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,217 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039211) roll requested 2024-11-24T13:47:19,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039217 2024-11-24T13:47:19,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039211 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039217 2024-11-24T13:47:19,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,223 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,224 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=333, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,224 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039217) roll requested 2024-11-24T13:47:19,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039224 2024-11-24T13:47:19,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039217 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039224 2024-11-24T13:47:19,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,231 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,231 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=334, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,231 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039224) roll requested 2024-11-24T13:47:19,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039232 2024-11-24T13:47:19,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039224 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039232 2024-11-24T13:47:19,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,239 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,239 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=335, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,239 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039232) roll requested 2024-11-24T13:47:19,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039239 2024-11-24T13:47:19,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039232 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039239 2024-11-24T13:47:19,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,246 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,246 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=336, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,246 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039239) roll requested 2024-11-24T13:47:19,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039246 2024-11-24T13:47:19,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039239 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039246 2024-11-24T13:47:19,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,252 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,252 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=337, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,252 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039246) roll requested 2024-11-24T13:47:19,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039252 2024-11-24T13:47:19,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039246 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039252 2024-11-24T13:47:19,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,258 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,258 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=338, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,259 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039252) roll requested 2024-11-24T13:47:19,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039259 2024-11-24T13:47:19,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039252 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039259 2024-11-24T13:47:19,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,265 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,265 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=339, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,265 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039259) roll requested 2024-11-24T13:47:19,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039265 2024-11-24T13:47:19,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039259 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039265 2024-11-24T13:47:19,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,273 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,274 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=340, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,274 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039265) roll requested 2024-11-24T13:47:19,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039274 2024-11-24T13:47:19,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039265 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039274 2024-11-24T13:47:19,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,282 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,282 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=341, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,282 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039274) roll requested 2024-11-24T13:47:19,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039282 2024-11-24T13:47:19,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039274 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039282 2024-11-24T13:47:19,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,290 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,290 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=342, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,290 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039282) roll requested 2024-11-24T13:47:19,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039291 2024-11-24T13:47:19,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039282 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039291 2024-11-24T13:47:19,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,299 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,299 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=343, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,300 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039291) roll requested 2024-11-24T13:47:19,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039300 2024-11-24T13:47:19,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039291 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039300 2024-11-24T13:47:19,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,309 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,309 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=344, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,309 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039300) roll requested 2024-11-24T13:47:19,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039309 2024-11-24T13:47:19,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039300 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039309 2024-11-24T13:47:19,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,317 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,317 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=345, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,317 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039309) roll requested 2024-11-24T13:47:19,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039317 2024-11-24T13:47:19,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039309 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039317 2024-11-24T13:47:19,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,324 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,324 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=346, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,324 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039317) roll requested 2024-11-24T13:47:19,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039324 2024-11-24T13:47:19,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039317 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039324 2024-11-24T13:47:19,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,333 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,333 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=347, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,333 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039324) roll requested 2024-11-24T13:47:19,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039333 2024-11-24T13:47:19,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039324 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039333 2024-11-24T13:47:19,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,341 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,341 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=348, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,342 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039333) roll requested 2024-11-24T13:47:19,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039342 2024-11-24T13:47:19,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039333 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039342 2024-11-24T13:47:19,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,351 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,351 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=349, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,351 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039342) roll requested 2024-11-24T13:47:19,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039351 2024-11-24T13:47:19,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039342 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039351 2024-11-24T13:47:19,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,361 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,361 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=350, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,361 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039351) roll requested 2024-11-24T13:47:19,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039362 2024-11-24T13:47:19,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039351 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039362 2024-11-24T13:47:19,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,372 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,372 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=351, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,372 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039362) roll requested 2024-11-24T13:47:19,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039372 2024-11-24T13:47:19,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039362 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039372 2024-11-24T13:47:19,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,378 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,379 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=352, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,379 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039372) roll requested 2024-11-24T13:47:19,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039379 2024-11-24T13:47:19,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039372 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039379 2024-11-24T13:47:19,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,384 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,384 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=353, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,384 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039379) roll requested 2024-11-24T13:47:19,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039384 2024-11-24T13:47:19,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039379 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039384 2024-11-24T13:47:19,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,392 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,392 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=354, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,392 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039384) roll requested 2024-11-24T13:47:19,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039392 2024-11-24T13:47:19,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039384 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039392 2024-11-24T13:47:19,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,399 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,399 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=355, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,400 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039392) roll requested 2024-11-24T13:47:19,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039400 2024-11-24T13:47:19,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039392 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039400 2024-11-24T13:47:19,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,408 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=356, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,408 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039400) roll requested 2024-11-24T13:47:19,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039409 2024-11-24T13:47:19,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039400 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039409 2024-11-24T13:47:19,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,416 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,417 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=357, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,417 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039409) roll requested 2024-11-24T13:47:19,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039417 2024-11-24T13:47:19,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039409 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039417 2024-11-24T13:47:19,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,426 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,427 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=358, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,427 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039417) roll requested 2024-11-24T13:47:19,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039427 2024-11-24T13:47:19,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039417 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039427 2024-11-24T13:47:19,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,436 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,436 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=359, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,436 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039427) roll requested 2024-11-24T13:47:19,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039436 2024-11-24T13:47:19,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039427 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039436 2024-11-24T13:47:19,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,444 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,444 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=360, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,444 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039436) roll requested 2024-11-24T13:47:19,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039444 2024-11-24T13:47:19,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039436 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039444 2024-11-24T13:47:19,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,452 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,452 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=361, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,453 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039444) roll requested 2024-11-24T13:47:19,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039453 2024-11-24T13:47:19,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039444 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039453 2024-11-24T13:47:19,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,460 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,461 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=362, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,461 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039453) roll requested 2024-11-24T13:47:19,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039461 2024-11-24T13:47:19,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039453 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039461 2024-11-24T13:47:19,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,468 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,468 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=363, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,468 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039461) roll requested 2024-11-24T13:47:19,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039469 2024-11-24T13:47:19,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039461 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039469 2024-11-24T13:47:19,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,477 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,477 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=364, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,478 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039469) roll requested 2024-11-24T13:47:19,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039478 2024-11-24T13:47:19,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039469 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039478 2024-11-24T13:47:19,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,487 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,488 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=365, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,488 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039478) roll requested 2024-11-24T13:47:19,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039488 2024-11-24T13:47:19,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039478 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039488 2024-11-24T13:47:19,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,498 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,499 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=366, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,499 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039488) roll requested 2024-11-24T13:47:19,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039499 2024-11-24T13:47:19,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039488 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039499 2024-11-24T13:47:19,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,509 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,509 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=367, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,509 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039499) roll requested 2024-11-24T13:47:19,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039509 2024-11-24T13:47:19,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039499 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039509 2024-11-24T13:47:19,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,516 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,516 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=368, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,517 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039509) roll requested 2024-11-24T13:47:19,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039517 2024-11-24T13:47:19,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039509 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039517 2024-11-24T13:47:19,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,525 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,525 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=369, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,525 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039517) roll requested 2024-11-24T13:47:19,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039525 2024-11-24T13:47:19,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039517 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039525 2024-11-24T13:47:19,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,533 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,533 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=370, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,534 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039525) roll requested 2024-11-24T13:47:19,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039534 2024-11-24T13:47:19,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039525 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039534 2024-11-24T13:47:19,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,541 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,542 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=371, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,542 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039534) roll requested 2024-11-24T13:47:19,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039542 2024-11-24T13:47:19,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039534 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039542 2024-11-24T13:47:19,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,551 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,551 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=372, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,551 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039542) roll requested 2024-11-24T13:47:19,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039551 2024-11-24T13:47:19,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039542 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039551 2024-11-24T13:47:19,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,558 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=373, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,558 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,558 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039551) roll requested 2024-11-24T13:47:19,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039558 2024-11-24T13:47:19,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039551 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039558 2024-11-24T13:47:19,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,566 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,566 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=374, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,566 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039558) roll requested 2024-11-24T13:47:19,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039566 2024-11-24T13:47:19,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039558 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039566 2024-11-24T13:47:19,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,574 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,574 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=375, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,574 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039566) roll requested 2024-11-24T13:47:19,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039575 2024-11-24T13:47:19,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039566 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039575 2024-11-24T13:47:19,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,582 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,582 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=376, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,583 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039575) roll requested 2024-11-24T13:47:19,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039583 2024-11-24T13:47:19,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039575 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039583 2024-11-24T13:47:19,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,591 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,591 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=377, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,591 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039583) roll requested 2024-11-24T13:47:19,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039591 2024-11-24T13:47:19,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039583 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039591 2024-11-24T13:47:19,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,600 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,600 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=378, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,600 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039591) roll requested 2024-11-24T13:47:19,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039600 2024-11-24T13:47:19,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039591 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039600 2024-11-24T13:47:19,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,609 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,610 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=379, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,610 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039600) roll requested 2024-11-24T13:47:19,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039610 2024-11-24T13:47:19,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039600 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039610 2024-11-24T13:47:19,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,618 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,618 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=380, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,618 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039610) roll requested 2024-11-24T13:47:19,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039619 2024-11-24T13:47:19,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039610 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039619 2024-11-24T13:47:19,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,627 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,627 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=381, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,627 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039619) roll requested 2024-11-24T13:47:19,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039627 2024-11-24T13:47:19,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039619 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039627 2024-11-24T13:47:19,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,635 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,635 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=382, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,635 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039627) roll requested 2024-11-24T13:47:19,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039635 2024-11-24T13:47:19,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039627 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039635 2024-11-24T13:47:19,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,643 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,643 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=383, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,643 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039635) roll requested 2024-11-24T13:47:19,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039643 2024-11-24T13:47:19,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039635 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039643 2024-11-24T13:47:19,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,651 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=384, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,651 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,651 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039643) roll requested 2024-11-24T13:47:19,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039652 2024-11-24T13:47:19,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039643 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039652 2024-11-24T13:47:19,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,659 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,659 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=385, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,660 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039652) roll requested 2024-11-24T13:47:19,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039660 2024-11-24T13:47:19,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039652 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039660 2024-11-24T13:47:19,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,666 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,666 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=386, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,667 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039660) roll requested 2024-11-24T13:47:19,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039667 2024-11-24T13:47:19,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039660 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039667 2024-11-24T13:47:19,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,674 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,674 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=387, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,674 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039667) roll requested 2024-11-24T13:47:19,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039674 2024-11-24T13:47:19,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039667 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039674 2024-11-24T13:47:19,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,681 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,681 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=388, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,682 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039674) roll requested 2024-11-24T13:47:19,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039682 2024-11-24T13:47:19,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039674 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039682 2024-11-24T13:47:19,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,690 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,690 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=389, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,690 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039682) roll requested 2024-11-24T13:47:19,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039691 2024-11-24T13:47:19,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039682 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039691 2024-11-24T13:47:19,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,698 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,698 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=390, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,698 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039691) roll requested 2024-11-24T13:47:19,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039698 2024-11-24T13:47:19,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039691 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039698 2024-11-24T13:47:19,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,705 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,705 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=391, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,706 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039698) roll requested 2024-11-24T13:47:19,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039706 2024-11-24T13:47:19,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039698 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039706 2024-11-24T13:47:19,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,712 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,713 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=392, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,713 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039706) roll requested 2024-11-24T13:47:19,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039713 2024-11-24T13:47:19,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039706 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039713 2024-11-24T13:47:19,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,721 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,721 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=393, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,722 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039713) roll requested 2024-11-24T13:47:19,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039722 2024-11-24T13:47:19,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039713 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039722 2024-11-24T13:47:19,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,730 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,730 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=394, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,730 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039722) roll requested 2024-11-24T13:47:19,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039730 2024-11-24T13:47:19,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039722 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039730 2024-11-24T13:47:19,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,739 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,739 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=395, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,739 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039730) roll requested 2024-11-24T13:47:19,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039739 2024-11-24T13:47:19,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039730 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039739 2024-11-24T13:47:19,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,745 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,745 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=396, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,745 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039739) roll requested 2024-11-24T13:47:19,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039745 2024-11-24T13:47:19,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039739 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039745 2024-11-24T13:47:19,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,751 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,751 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=397, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,751 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039745) roll requested 2024-11-24T13:47:19,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039751 2024-11-24T13:47:19,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039745 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039751 2024-11-24T13:47:19,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,757 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,757 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=398, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,757 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039751) roll requested 2024-11-24T13:47:19,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039757 2024-11-24T13:47:19,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039751 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039757 2024-11-24T13:47:19,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,764 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,764 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=399, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,764 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039757) roll requested 2024-11-24T13:47:19,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039764 2024-11-24T13:47:19,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039757 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039764 2024-11-24T13:47:19,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,771 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,771 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=400, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,771 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039764) roll requested 2024-11-24T13:47:19,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039772 2024-11-24T13:47:19,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039764 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039772 2024-11-24T13:47:19,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,779 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,780 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=401, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,780 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039772) roll requested 2024-11-24T13:47:19,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039780 2024-11-24T13:47:19,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039772 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039780 2024-11-24T13:47:19,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,786 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,786 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=402, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,786 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039780) roll requested 2024-11-24T13:47:19,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039786 2024-11-24T13:47:19,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039780 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039786 2024-11-24T13:47:19,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,794 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,794 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=403, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,794 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039786) roll requested 2024-11-24T13:47:19,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039794 2024-11-24T13:47:19,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039786 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039794 2024-11-24T13:47:19,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,802 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,802 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=404, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,802 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039794) roll requested 2024-11-24T13:47:19,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039802 2024-11-24T13:47:19,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039794 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039802 2024-11-24T13:47:19,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,810 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,810 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=405, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,811 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039802) roll requested 2024-11-24T13:47:19,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039811 2024-11-24T13:47:19,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039802 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039811 2024-11-24T13:47:19,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,820 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,820 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=406, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,821 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039811) roll requested 2024-11-24T13:47:19,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039821 2024-11-24T13:47:19,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039811 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039821 2024-11-24T13:47:19,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,827 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,827 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=407, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,827 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039821) roll requested 2024-11-24T13:47:19,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039827 2024-11-24T13:47:19,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039821 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039827 2024-11-24T13:47:19,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,834 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,834 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=408, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,834 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039827) roll requested 2024-11-24T13:47:19,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039834 2024-11-24T13:47:19,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039827 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039834 2024-11-24T13:47:19,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,842 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,842 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=409, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,842 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039834) roll requested 2024-11-24T13:47:19,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039843 2024-11-24T13:47:19,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039834 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039843 2024-11-24T13:47:19,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,851 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,851 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=410, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,851 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039843) roll requested 2024-11-24T13:47:19,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039851 2024-11-24T13:47:19,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039843 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039851 2024-11-24T13:47:19,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,859 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,859 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=411, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,859 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039851) roll requested 2024-11-24T13:47:19,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039859 2024-11-24T13:47:19,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039851 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039859 2024-11-24T13:47:19,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,866 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,867 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=412, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,867 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039859) roll requested 2024-11-24T13:47:19,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039867 2024-11-24T13:47:19,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039859 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039867 2024-11-24T13:47:19,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,874 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,874 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=413, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,875 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039867) roll requested 2024-11-24T13:47:19,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039875 2024-11-24T13:47:19,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039867 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039875 2024-11-24T13:47:19,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,882 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,882 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=414, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,882 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039875) roll requested 2024-11-24T13:47:19,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039882 2024-11-24T13:47:19,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039875 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039882 2024-11-24T13:47:19,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,889 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,889 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=415, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,890 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039882) roll requested 2024-11-24T13:47:19,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039890 2024-11-24T13:47:19,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039882 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039890 2024-11-24T13:47:19,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,898 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,898 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=416, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,898 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039890) roll requested 2024-11-24T13:47:19,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039899 2024-11-24T13:47:19,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039890 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039899 2024-11-24T13:47:19,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,905 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,906 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=417, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,906 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039899) roll requested 2024-11-24T13:47:19,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039906 2024-11-24T13:47:19,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039899 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039906 2024-11-24T13:47:19,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,912 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,912 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=418, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,912 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039906) roll requested 2024-11-24T13:47:19,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039912 2024-11-24T13:47:19,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039906 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039912 2024-11-24T13:47:19,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,920 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,920 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=419, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,920 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039912) roll requested 2024-11-24T13:47:19,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039920 2024-11-24T13:47:19,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039912 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039920 2024-11-24T13:47:19,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,929 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,929 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=420, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,929 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039920) roll requested 2024-11-24T13:47:19,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039929 2024-11-24T13:47:19,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039920 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039929 2024-11-24T13:47:19,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,937 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,938 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=421, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,938 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039929) roll requested 2024-11-24T13:47:19,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039938 2024-11-24T13:47:19,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039929 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039938 2024-11-24T13:47:19,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,946 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,946 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=422, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,946 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039938) roll requested 2024-11-24T13:47:19,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039946 2024-11-24T13:47:19,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039938 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039946 2024-11-24T13:47:19,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,954 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,954 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=423, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,954 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039946) roll requested 2024-11-24T13:47:19,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039955 2024-11-24T13:47:19,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039946 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039955 2024-11-24T13:47:19,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,960 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,961 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=424, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,961 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039955) roll requested 2024-11-24T13:47:19,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039961 2024-11-24T13:47:19,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039955 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039961 2024-11-24T13:47:19,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,967 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,968 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=425, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,968 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039961) roll requested 2024-11-24T13:47:19,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039968 2024-11-24T13:47:19,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039961 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039968 2024-11-24T13:47:19,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,977 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,977 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=426, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,977 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039968) roll requested 2024-11-24T13:47:19,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039977 2024-11-24T13:47:19,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039968 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039977 2024-11-24T13:47:19,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,986 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,986 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=427, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,987 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039977) roll requested 2024-11-24T13:47:19,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039987 2024-11-24T13:47:19,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:19,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039977 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039987 2024-11-24T13:47:19,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:19,997 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:19,997 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=428, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:19,997 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:19,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039987) roll requested 2024-11-24T13:47:19,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456039997 2024-11-24T13:47:20,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039987 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039997 2024-11-24T13:47:20,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,005 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,005 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=429, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,005 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456039997) roll requested 2024-11-24T13:47:20,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040005 2024-11-24T13:47:20,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456039997 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040005 2024-11-24T13:47:20,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,013 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,013 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=430, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,013 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040005) roll requested 2024-11-24T13:47:20,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040013 2024-11-24T13:47:20,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040005 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040013 2024-11-24T13:47:20,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,020 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,020 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=431, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,020 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040013) roll requested 2024-11-24T13:47:20,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040020 2024-11-24T13:47:20,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040013 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040020 2024-11-24T13:47:20,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,026 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,026 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=432, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,026 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040020) roll requested 2024-11-24T13:47:20,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040026 2024-11-24T13:47:20,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040020 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040026 2024-11-24T13:47:20,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,033 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,033 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=433, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,033 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040026) roll requested 2024-11-24T13:47:20,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040033 2024-11-24T13:47:20,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040026 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040033 2024-11-24T13:47:20,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,041 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,041 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=434, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,041 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040033) roll requested 2024-11-24T13:47:20,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040041 2024-11-24T13:47:20,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040033 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040041 2024-11-24T13:47:20,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,049 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,050 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=435, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,050 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040041) roll requested 2024-11-24T13:47:20,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040050 2024-11-24T13:47:20,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040041 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040050 2024-11-24T13:47:20,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,058 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,058 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=436, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,059 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040050) roll requested 2024-11-24T13:47:20,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040059 2024-11-24T13:47:20,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040050 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040059 2024-11-24T13:47:20,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,065 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,065 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=437, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,066 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040059) roll requested 2024-11-24T13:47:20,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040066 2024-11-24T13:47:20,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040059 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040066 2024-11-24T13:47:20,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,072 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,072 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=438, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,072 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040066) roll requested 2024-11-24T13:47:20,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040072 2024-11-24T13:47:20,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040066 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040072 2024-11-24T13:47:20,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,083 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,083 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=439, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,084 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040072) roll requested 2024-11-24T13:47:20,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040084 2024-11-24T13:47:20,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040072 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040084 2024-11-24T13:47:20,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,089 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=440, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,089 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,089 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040084) roll requested 2024-11-24T13:47:20,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040089 2024-11-24T13:47:20,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040084 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040089 2024-11-24T13:47:20,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,095 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=441, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,095 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,095 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040089) roll requested 2024-11-24T13:47:20,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040095 2024-11-24T13:47:20,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040089 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040095 2024-11-24T13:47:20,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,102 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,102 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=442, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,102 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040095) roll requested 2024-11-24T13:47:20,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040102 2024-11-24T13:47:20,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040095 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040102 2024-11-24T13:47:20,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,110 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,110 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=443, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,110 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040102) roll requested 2024-11-24T13:47:20,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040110 2024-11-24T13:47:20,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040102 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040110 2024-11-24T13:47:20,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,119 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,119 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=444, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,119 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040110) roll requested 2024-11-24T13:47:20,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040119 2024-11-24T13:47:20,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040110 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040119 2024-11-24T13:47:20,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,128 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,129 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=445, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,129 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040119) roll requested 2024-11-24T13:47:20,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040129 2024-11-24T13:47:20,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040119 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040129 2024-11-24T13:47:20,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,138 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,138 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=446, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,139 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040129) roll requested 2024-11-24T13:47:20,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040139 2024-11-24T13:47:20,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040129 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040139 2024-11-24T13:47:20,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,147 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,147 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=447, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,147 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040139) roll requested 2024-11-24T13:47:20,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040147 2024-11-24T13:47:20,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040139 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040147 2024-11-24T13:47:20,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,153 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,153 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=448, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,153 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040147) roll requested 2024-11-24T13:47:20,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040153 2024-11-24T13:47:20,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040147 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040153 2024-11-24T13:47:20,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,161 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,161 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=449, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,161 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040153) roll requested 2024-11-24T13:47:20,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040161 2024-11-24T13:47:20,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040153 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040161 2024-11-24T13:47:20,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,170 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,170 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=450, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,170 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040161) roll requested 2024-11-24T13:47:20,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040170 2024-11-24T13:47:20,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040161 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040170 2024-11-24T13:47:20,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,178 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,178 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=451, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,178 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040170) roll requested 2024-11-24T13:47:20,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040178 2024-11-24T13:47:20,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040170 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040178 2024-11-24T13:47:20,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,187 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,187 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=452, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,187 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040178) roll requested 2024-11-24T13:47:20,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040187 2024-11-24T13:47:20,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040178 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040187 2024-11-24T13:47:20,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,195 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,196 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=453, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,196 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040187) roll requested 2024-11-24T13:47:20,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040196 2024-11-24T13:47:20,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040187 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040196 2024-11-24T13:47:20,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,202 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,203 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=454, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,203 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040196) roll requested 2024-11-24T13:47:20,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040203 2024-11-24T13:47:20,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040196 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040203 2024-11-24T13:47:20,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,209 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,209 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=455, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,210 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040203) roll requested 2024-11-24T13:47:20,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040210 2024-11-24T13:47:20,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040203 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040210 2024-11-24T13:47:20,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,219 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,219 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=456, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,219 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040210) roll requested 2024-11-24T13:47:20,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040219 2024-11-24T13:47:20,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040210 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040219 2024-11-24T13:47:20,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,228 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,228 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=457, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,228 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040219) roll requested 2024-11-24T13:47:20,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040228 2024-11-24T13:47:20,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040219 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040228 2024-11-24T13:47:20,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,236 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,236 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=458, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,237 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040228) roll requested 2024-11-24T13:47:20,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040237 2024-11-24T13:47:20,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040228 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040237 2024-11-24T13:47:20,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,243 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,243 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=459, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,243 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040237) roll requested 2024-11-24T13:47:20,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040243 2024-11-24T13:47:20,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040237 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040243 2024-11-24T13:47:20,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,249 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,249 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=460, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,250 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040243) roll requested 2024-11-24T13:47:20,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040250 2024-11-24T13:47:20,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040243 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040250 2024-11-24T13:47:20,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,257 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,257 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=461, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,257 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040250) roll requested 2024-11-24T13:47:20,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040257 2024-11-24T13:47:20,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040250 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040257 2024-11-24T13:47:20,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,266 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,266 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=462, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,266 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040257) roll requested 2024-11-24T13:47:20,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040266 2024-11-24T13:47:20,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040257 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040266 2024-11-24T13:47:20,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,276 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,276 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=463, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,276 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040266) roll requested 2024-11-24T13:47:20,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040276 2024-11-24T13:47:20,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040266 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040276 2024-11-24T13:47:20,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,283 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,283 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=464, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,283 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040276) roll requested 2024-11-24T13:47:20,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040283 2024-11-24T13:47:20,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040276 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040283 2024-11-24T13:47:20,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,290 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,290 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=465, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,290 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040283) roll requested 2024-11-24T13:47:20,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040290 2024-11-24T13:47:20,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040283 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040290 2024-11-24T13:47:20,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,297 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,297 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=466, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,297 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040290) roll requested 2024-11-24T13:47:20,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040297 2024-11-24T13:47:20,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040290 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040297 2024-11-24T13:47:20,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,304 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,304 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=467, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,305 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040297) roll requested 2024-11-24T13:47:20,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040305 2024-11-24T13:47:20,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040297 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040305 2024-11-24T13:47:20,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,312 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,312 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=468, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,312 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040305) roll requested 2024-11-24T13:47:20,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040313 2024-11-24T13:47:20,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040305 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040313 2024-11-24T13:47:20,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,319 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,319 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=469, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,320 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040313) roll requested 2024-11-24T13:47:20,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040320 2024-11-24T13:47:20,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040313 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040320 2024-11-24T13:47:20,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,327 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,328 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=470, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,328 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040320) roll requested 2024-11-24T13:47:20,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040328 2024-11-24T13:47:20,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040320 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040328 2024-11-24T13:47:20,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,335 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,335 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=471, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,335 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040328) roll requested 2024-11-24T13:47:20,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040336 2024-11-24T13:47:20,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040328 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040336 2024-11-24T13:47:20,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,344 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,344 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=472, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,344 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040336) roll requested 2024-11-24T13:47:20,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040344 2024-11-24T13:47:20,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040336 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040344 2024-11-24T13:47:20,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,352 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,352 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=473, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,352 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040344) roll requested 2024-11-24T13:47:20,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040352 2024-11-24T13:47:20,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040344 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040352 2024-11-24T13:47:20,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,360 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,360 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=474, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,361 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040352) roll requested 2024-11-24T13:47:20,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040361 2024-11-24T13:47:20,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040352 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040361 2024-11-24T13:47:20,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,368 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,369 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=475, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,369 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040361) roll requested 2024-11-24T13:47:20,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040369 2024-11-24T13:47:20,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040361 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040369 2024-11-24T13:47:20,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,375 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,375 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=476, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,375 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040369) roll requested 2024-11-24T13:47:20,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040375 2024-11-24T13:47:20,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040369 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040375 2024-11-24T13:47:20,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,381 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,381 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=477, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,381 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040375) roll requested 2024-11-24T13:47:20,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040381 2024-11-24T13:47:20,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040375 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040381 2024-11-24T13:47:20,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,389 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,389 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=478, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,389 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040381) roll requested 2024-11-24T13:47:20,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040389 2024-11-24T13:47:20,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040381 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040389 2024-11-24T13:47:20,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,397 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,397 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=479, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,397 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040389) roll requested 2024-11-24T13:47:20,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040397 2024-11-24T13:47:20,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040389 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040397 2024-11-24T13:47:20,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,405 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,405 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=480, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,405 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040397) roll requested 2024-11-24T13:47:20,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040405 2024-11-24T13:47:20,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040397 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040405 2024-11-24T13:47:20,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,413 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,413 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=481, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,413 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040405) roll requested 2024-11-24T13:47:20,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040413 2024-11-24T13:47:20,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040405 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040413 2024-11-24T13:47:20,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,423 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,423 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=482, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,423 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040413) roll requested 2024-11-24T13:47:20,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040423 2024-11-24T13:47:20,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040413 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040423 2024-11-24T13:47:20,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,430 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,430 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=483, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,430 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040423) roll requested 2024-11-24T13:47:20,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040430 2024-11-24T13:47:20,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040423 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040430 2024-11-24T13:47:20,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,436 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,436 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=484, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,436 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040430) roll requested 2024-11-24T13:47:20,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040436 2024-11-24T13:47:20,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040430 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040436 2024-11-24T13:47:20,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,442 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,442 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=485, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,442 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040436) roll requested 2024-11-24T13:47:20,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040442 2024-11-24T13:47:20,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040436 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040442 2024-11-24T13:47:20,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,447 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,447 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=486, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,447 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040442) roll requested 2024-11-24T13:47:20,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040447 2024-11-24T13:47:20,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040442 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040447 2024-11-24T13:47:20,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,455 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,455 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=487, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,455 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040447) roll requested 2024-11-24T13:47:20,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040455 2024-11-24T13:47:20,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040447 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040455 2024-11-24T13:47:20,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,464 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,464 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=488, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,464 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040455) roll requested 2024-11-24T13:47:20,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040464 2024-11-24T13:47:20,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040455 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040464 2024-11-24T13:47:20,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,472 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,472 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=489, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,472 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040464) roll requested 2024-11-24T13:47:20,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040472 2024-11-24T13:47:20,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040464 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040472 2024-11-24T13:47:20,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,477 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,477 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=490, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,477 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040472) roll requested 2024-11-24T13:47:20,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040477 2024-11-24T13:47:20,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040472 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040477 2024-11-24T13:47:20,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,482 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,482 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=491, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,482 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040477) roll requested 2024-11-24T13:47:20,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040482 2024-11-24T13:47:20,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040477 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040482 2024-11-24T13:47:20,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,488 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,488 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=492, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,488 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040482) roll requested 2024-11-24T13:47:20,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040488 2024-11-24T13:47:20,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040482 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040488 2024-11-24T13:47:20,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,494 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,494 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=493, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,494 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040488) roll requested 2024-11-24T13:47:20,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040494 2024-11-24T13:47:20,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040488 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040494 2024-11-24T13:47:20,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,501 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,501 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=494, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,501 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040494) roll requested 2024-11-24T13:47:20,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040501 2024-11-24T13:47:20,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040494 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040501 2024-11-24T13:47:20,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,508 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,508 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=495, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,508 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040501) roll requested 2024-11-24T13:47:20,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040508 2024-11-24T13:47:20,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040501 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040508 2024-11-24T13:47:20,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,515 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,515 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=496, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,515 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040508) roll requested 2024-11-24T13:47:20,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040515 2024-11-24T13:47:20,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040508 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040515 2024-11-24T13:47:20,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,524 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,524 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=497, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,524 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040515) roll requested 2024-11-24T13:47:20,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040524 2024-11-24T13:47:20,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040515 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040524 2024-11-24T13:47:20,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,531 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,532 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=498, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,532 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040524) roll requested 2024-11-24T13:47:20,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040532 2024-11-24T13:47:20,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040524 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040532 2024-11-24T13:47:20,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,540 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,540 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=499, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,541 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040532) roll requested 2024-11-24T13:47:20,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040541 2024-11-24T13:47:20,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040532 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040541 2024-11-24T13:47:20,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,550 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,550 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=500, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,550 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040541) roll requested 2024-11-24T13:47:20,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040550 2024-11-24T13:47:20,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040541 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040550 2024-11-24T13:47:20,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,559 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,559 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=501, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,560 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040550) roll requested 2024-11-24T13:47:20,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040560 2024-11-24T13:47:20,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040550 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040560 2024-11-24T13:47:20,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,569 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,569 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=502, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,569 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040560) roll requested 2024-11-24T13:47:20,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040569 2024-11-24T13:47:20,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040560 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040569 2024-11-24T13:47:20,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,580 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,580 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=503, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,580 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040569) roll requested 2024-11-24T13:47:20,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040580 2024-11-24T13:47:20,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040569 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040580 2024-11-24T13:47:20,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,590 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,590 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=504, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,590 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040580) roll requested 2024-11-24T13:47:20,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040590 2024-11-24T13:47:20,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040580 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040590 2024-11-24T13:47:20,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,598 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,598 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=505, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,598 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040590) roll requested 2024-11-24T13:47:20,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040598 2024-11-24T13:47:20,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040590 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040598 2024-11-24T13:47:20,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,607 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,607 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=506, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,608 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040598) roll requested 2024-11-24T13:47:20,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040608 2024-11-24T13:47:20,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040598 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040608 2024-11-24T13:47:20,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,618 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,618 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=507, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,618 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040608) roll requested 2024-11-24T13:47:20,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040618 2024-11-24T13:47:20,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040608 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040618 2024-11-24T13:47:20,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,628 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,629 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=508, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,629 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040618) roll requested 2024-11-24T13:47:20,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040629 2024-11-24T13:47:20,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040618 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040629 2024-11-24T13:47:20,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,638 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,638 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=509, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,638 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040629) roll requested 2024-11-24T13:47:20,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040638 2024-11-24T13:47:20,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040629 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040638 2024-11-24T13:47:20,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,643 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,644 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=510, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,644 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040638) roll requested 2024-11-24T13:47:20,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040644 2024-11-24T13:47:20,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040638 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040644 2024-11-24T13:47:20,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,649 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,649 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=511, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,650 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040644) roll requested 2024-11-24T13:47:20,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040650 2024-11-24T13:47:20,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040644 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040650 2024-11-24T13:47:20,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,657 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,657 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=512, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,657 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040650) roll requested 2024-11-24T13:47:20,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040657 2024-11-24T13:47:20,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040650 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040657 2024-11-24T13:47:20,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,665 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,665 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=513, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,666 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040657) roll requested 2024-11-24T13:47:20,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040666 2024-11-24T13:47:20,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040657 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040666 2024-11-24T13:47:20,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,674 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,674 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=514, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,675 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040666) roll requested 2024-11-24T13:47:20,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040675 2024-11-24T13:47:20,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040666 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040675 2024-11-24T13:47:20,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,683 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,683 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=515, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,683 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040675) roll requested 2024-11-24T13:47:20,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040683 2024-11-24T13:47:20,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040675 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040683 2024-11-24T13:47:20,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,691 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,691 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=516, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,691 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040683) roll requested 2024-11-24T13:47:20,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040692 2024-11-24T13:47:20,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040683 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040692 2024-11-24T13:47:20,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,700 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,700 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=517, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,701 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040692) roll requested 2024-11-24T13:47:20,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040701 2024-11-24T13:47:20,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040692 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040701 2024-11-24T13:47:20,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,709 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=518, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040701) roll requested 2024-11-24T13:47:20,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040709 2024-11-24T13:47:20,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040709 2024-11-24T13:47:20,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,716 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,716 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=519, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,716 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040709) roll requested 2024-11-24T13:47:20,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040716 2024-11-24T13:47:20,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040716 2024-11-24T13:47:20,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,725 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,726 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=520, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,726 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040716) roll requested 2024-11-24T13:47:20,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040726 2024-11-24T13:47:20,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040716 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040726 2024-11-24T13:47:20,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,734 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,735 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=521, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,735 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040726) roll requested 2024-11-24T13:47:20,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040735 2024-11-24T13:47:20,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040726 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040735 2024-11-24T13:47:20,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,743 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,743 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=522, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,743 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040735) roll requested 2024-11-24T13:47:20,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040743 2024-11-24T13:47:20,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040735 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040743 2024-11-24T13:47:20,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,752 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,752 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=523, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,752 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040743) roll requested 2024-11-24T13:47:20,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040752 2024-11-24T13:47:20,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040743 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040752 2024-11-24T13:47:20,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,760 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,760 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=524, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,760 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040752) roll requested 2024-11-24T13:47:20,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040760 2024-11-24T13:47:20,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040752 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040760 2024-11-24T13:47:20,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,767 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,767 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=525, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,767 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040760) roll requested 2024-11-24T13:47:20,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040767 2024-11-24T13:47:20,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040760 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040767 2024-11-24T13:47:20,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,775 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,775 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=526, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,776 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040767) roll requested 2024-11-24T13:47:20,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040776 2024-11-24T13:47:20,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040767 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040776 2024-11-24T13:47:20,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,782 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,782 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=527, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,782 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040776) roll requested 2024-11-24T13:47:20,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040782 2024-11-24T13:47:20,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040776 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040782 2024-11-24T13:47:20,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,787 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,787 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=528, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,787 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040782) roll requested 2024-11-24T13:47:20,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040787 2024-11-24T13:47:20,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040782 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040787 2024-11-24T13:47:20,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,792 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,792 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=529, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,792 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040787) roll requested 2024-11-24T13:47:20,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040792 2024-11-24T13:47:20,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040787 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040792 2024-11-24T13:47:20,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,799 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,799 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=530, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,799 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040792) roll requested 2024-11-24T13:47:20,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040799 2024-11-24T13:47:20,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040792 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040799 2024-11-24T13:47:20,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,806 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,806 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=531, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,806 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040799) roll requested 2024-11-24T13:47:20,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040807 2024-11-24T13:47:20,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040799 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040807 2024-11-24T13:47:20,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,814 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,814 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=532, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,815 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040807) roll requested 2024-11-24T13:47:20,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040815 2024-11-24T13:47:20,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040807 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040815 2024-11-24T13:47:20,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,823 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,823 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=533, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,823 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040815) roll requested 2024-11-24T13:47:20,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040823 2024-11-24T13:47:20,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040815 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040823 2024-11-24T13:47:20,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,830 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,830 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=534, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,830 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040823) roll requested 2024-11-24T13:47:20,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040830 2024-11-24T13:47:20,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040823 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040830 2024-11-24T13:47:20,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,838 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,838 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=535, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,838 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040830) roll requested 2024-11-24T13:47:20,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040838 2024-11-24T13:47:20,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040830 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040838 2024-11-24T13:47:20,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,846 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,846 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=536, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,846 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040838) roll requested 2024-11-24T13:47:20,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040846 2024-11-24T13:47:20,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040838 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040846 2024-11-24T13:47:20,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,855 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,855 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=537, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,855 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040846) roll requested 2024-11-24T13:47:20,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040856 2024-11-24T13:47:20,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040846 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040856 2024-11-24T13:47:20,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,863 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,863 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=538, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,863 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040856) roll requested 2024-11-24T13:47:20,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040863 2024-11-24T13:47:20,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040856 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040863 2024-11-24T13:47:20,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,868 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,868 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=539, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,868 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040863) roll requested 2024-11-24T13:47:20,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040868 2024-11-24T13:47:20,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040863 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040868 2024-11-24T13:47:20,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,875 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,875 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=540, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,875 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040868) roll requested 2024-11-24T13:47:20,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040875 2024-11-24T13:47:20,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040868 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040875 2024-11-24T13:47:20,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,882 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,882 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=541, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,882 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040875) roll requested 2024-11-24T13:47:20,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040882 2024-11-24T13:47:20,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040875 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040882 2024-11-24T13:47:20,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,890 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,890 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=542, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,890 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040882) roll requested 2024-11-24T13:47:20,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040891 2024-11-24T13:47:20,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040882 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040891 2024-11-24T13:47:20,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,898 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,899 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=543, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,899 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040891) roll requested 2024-11-24T13:47:20,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040899 2024-11-24T13:47:20,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040891 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040899 2024-11-24T13:47:20,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,908 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,908 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=544, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,908 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040899) roll requested 2024-11-24T13:47:20,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040908 2024-11-24T13:47:20,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040899 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040908 2024-11-24T13:47:20,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,914 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,914 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=545, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,914 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040908) roll requested 2024-11-24T13:47:20,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040914 2024-11-24T13:47:20,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040908 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040914 2024-11-24T13:47:20,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,921 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,921 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=546, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,921 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040914) roll requested 2024-11-24T13:47:20,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040921 2024-11-24T13:47:20,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040914 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040921 2024-11-24T13:47:20,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,929 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,929 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=547, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,929 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040921) roll requested 2024-11-24T13:47:20,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040929 2024-11-24T13:47:20,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040921 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040929 2024-11-24T13:47:20,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,939 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,939 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=548, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,939 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040929) roll requested 2024-11-24T13:47:20,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040940 2024-11-24T13:47:20,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040929 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040940 2024-11-24T13:47:20,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,947 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,947 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=549, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,948 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040940) roll requested 2024-11-24T13:47:20,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040948 2024-11-24T13:47:20,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040940 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040948 2024-11-24T13:47:20,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,956 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,956 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=550, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,956 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040948) roll requested 2024-11-24T13:47:20,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040956 2024-11-24T13:47:20,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040948 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040956 2024-11-24T13:47:20,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,965 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,965 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=551, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,965 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040956) roll requested 2024-11-24T13:47:20,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040965 2024-11-24T13:47:20,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040956 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040965 2024-11-24T13:47:20,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,974 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,974 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=552, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,974 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040965) roll requested 2024-11-24T13:47:20,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040974 2024-11-24T13:47:20,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040965 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040974 2024-11-24T13:47:20,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,984 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,984 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=553, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,984 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040974) roll requested 2024-11-24T13:47:20,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040984 2024-11-24T13:47:20,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040974 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040984 2024-11-24T13:47:20,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:20,993 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:20,993 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=554, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:20,993 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:20,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040984) roll requested 2024-11-24T13:47:20,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456040993 2024-11-24T13:47:20,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:20,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040984 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040993 2024-11-24T13:47:21,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,000 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,000 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=555, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,000 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456040993) roll requested 2024-11-24T13:47:21,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041000 2024-11-24T13:47:21,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456040993 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041000 2024-11-24T13:47:21,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,006 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,007 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=556, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,007 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041000) roll requested 2024-11-24T13:47:21,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041007 2024-11-24T13:47:21,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041000 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041007 2024-11-24T13:47:21,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,016 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,017 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=557, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,017 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041007) roll requested 2024-11-24T13:47:21,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041017 2024-11-24T13:47:21,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041007 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041017 2024-11-24T13:47:21,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,024 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,024 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=558, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,024 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041017) roll requested 2024-11-24T13:47:21,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041024 2024-11-24T13:47:21,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041017 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041024 2024-11-24T13:47:21,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,029 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,029 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=559, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,029 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041024) roll requested 2024-11-24T13:47:21,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041030 2024-11-24T13:47:21,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041024 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041030 2024-11-24T13:47:21,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,035 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,035 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=560, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,036 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041030) roll requested 2024-11-24T13:47:21,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041036 2024-11-24T13:47:21,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041030 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041036 2024-11-24T13:47:21,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,042 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,043 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=561, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,043 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041036) roll requested 2024-11-24T13:47:21,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041043 2024-11-24T13:47:21,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041036 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041043 2024-11-24T13:47:21,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,050 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,051 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=562, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,051 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041043) roll requested 2024-11-24T13:47:21,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041051 2024-11-24T13:47:21,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041043 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041051 2024-11-24T13:47:21,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,056 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,056 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=563, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,056 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041051) roll requested 2024-11-24T13:47:21,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041056 2024-11-24T13:47:21,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041051 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041056 2024-11-24T13:47:21,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,061 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,061 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=564, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,061 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041056) roll requested 2024-11-24T13:47:21,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041061 2024-11-24T13:47:21,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041056 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041061 2024-11-24T13:47:21,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,067 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,068 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=565, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,068 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041061) roll requested 2024-11-24T13:47:21,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041068 2024-11-24T13:47:21,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041061 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041068 2024-11-24T13:47:21,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,077 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,078 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=566, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,078 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041068) roll requested 2024-11-24T13:47:21,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041078 2024-11-24T13:47:21,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041068 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041078 2024-11-24T13:47:21,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,083 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,083 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=567, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,083 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041078) roll requested 2024-11-24T13:47:21,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041083 2024-11-24T13:47:21,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041078 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041083 2024-11-24T13:47:21,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,088 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,088 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=568, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,089 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041083) roll requested 2024-11-24T13:47:21,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041089 2024-11-24T13:47:21,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041083 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041089 2024-11-24T13:47:21,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,094 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,094 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=569, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,095 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041089) roll requested 2024-11-24T13:47:21,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041095 2024-11-24T13:47:21,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041089 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041095 2024-11-24T13:47:21,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,102 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,102 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=570, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,102 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041095) roll requested 2024-11-24T13:47:21,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041102 2024-11-24T13:47:21,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041095 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041102 2024-11-24T13:47:21,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,110 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,110 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=571, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,110 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041102) roll requested 2024-11-24T13:47:21,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041110 2024-11-24T13:47:21,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041102 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041110 2024-11-24T13:47:21,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,121 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,121 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=572, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,121 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041110) roll requested 2024-11-24T13:47:21,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041121 2024-11-24T13:47:21,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041110 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041121 2024-11-24T13:47:21,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,129 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,130 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=573, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,130 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041121) roll requested 2024-11-24T13:47:21,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041130 2024-11-24T13:47:21,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041121 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041130 2024-11-24T13:47:21,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,138 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,138 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=574, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,138 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041130) roll requested 2024-11-24T13:47:21,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041138 2024-11-24T13:47:21,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041130 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041138 2024-11-24T13:47:21,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,147 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,147 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=575, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,147 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041138) roll requested 2024-11-24T13:47:21,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041147 2024-11-24T13:47:21,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041138 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041147 2024-11-24T13:47:21,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,153 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,153 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=576, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,153 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041147) roll requested 2024-11-24T13:47:21,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041153 2024-11-24T13:47:21,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041147 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041153 2024-11-24T13:47:21,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,159 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,159 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=577, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,160 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041153) roll requested 2024-11-24T13:47:21,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041160 2024-11-24T13:47:21,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041153 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041160 2024-11-24T13:47:21,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,167 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,167 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=578, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,168 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041160) roll requested 2024-11-24T13:47:21,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041168 2024-11-24T13:47:21,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041160 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041168 2024-11-24T13:47:21,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,179 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,179 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=579, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,179 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041168) roll requested 2024-11-24T13:47:21,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041179 2024-11-24T13:47:21,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041168 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041179 2024-11-24T13:47:21,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,191 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,191 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=580, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,191 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041179) roll requested 2024-11-24T13:47:21,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041191 2024-11-24T13:47:21,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041179 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041191 2024-11-24T13:47:21,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,197 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,197 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=581, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,198 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041191) roll requested 2024-11-24T13:47:21,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041198 2024-11-24T13:47:21,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041191 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041198 2024-11-24T13:47:21,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,208 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,208 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=582, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,208 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041198) roll requested 2024-11-24T13:47:21,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041208 2024-11-24T13:47:21,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041198 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041208 2024-11-24T13:47:21,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,217 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,217 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=583, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,217 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041208) roll requested 2024-11-24T13:47:21,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041217 2024-11-24T13:47:21,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041208 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041217 2024-11-24T13:47:21,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,222 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,222 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=584, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,222 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041217) roll requested 2024-11-24T13:47:21,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041223 2024-11-24T13:47:21,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041217 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041223 2024-11-24T13:47:21,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,228 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,228 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=585, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,228 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041223) roll requested 2024-11-24T13:47:21,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041228 2024-11-24T13:47:21,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041223 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041228 2024-11-24T13:47:21,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,236 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,236 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=586, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,236 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041228) roll requested 2024-11-24T13:47:21,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041236 2024-11-24T13:47:21,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041228 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041236 2024-11-24T13:47:21,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,243 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,243 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=587, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,243 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041236) roll requested 2024-11-24T13:47:21,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041243 2024-11-24T13:47:21,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041236 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041243 2024-11-24T13:47:21,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,251 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,252 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=588, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,252 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041243) roll requested 2024-11-24T13:47:21,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041252 2024-11-24T13:47:21,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041243 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041252 2024-11-24T13:47:21,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,259 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,260 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=589, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,260 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041252) roll requested 2024-11-24T13:47:21,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041260 2024-11-24T13:47:21,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041252 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041260 2024-11-24T13:47:21,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,267 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,267 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=590, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,268 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041260) roll requested 2024-11-24T13:47:21,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041268 2024-11-24T13:47:21,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041260 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041268 2024-11-24T13:47:21,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,276 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,276 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=591, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,277 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041268) roll requested 2024-11-24T13:47:21,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041277 2024-11-24T13:47:21,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041268 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041277 2024-11-24T13:47:21,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,286 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,286 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=592, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,286 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041277) roll requested 2024-11-24T13:47:21,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041286 2024-11-24T13:47:21,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041286 2024-11-24T13:47:21,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,294 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,294 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=593, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,295 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041286) roll requested 2024-11-24T13:47:21,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041295 2024-11-24T13:47:21,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041286 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041295 2024-11-24T13:47:21,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,302 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,302 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=594, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,302 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041295) roll requested 2024-11-24T13:47:21,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041302 2024-11-24T13:47:21,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041295 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041302 2024-11-24T13:47:21,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,309 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,309 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=595, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,309 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041302) roll requested 2024-11-24T13:47:21,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041309 2024-11-24T13:47:21,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041302 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041309 2024-11-24T13:47:21,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,316 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,316 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=596, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,317 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041309) roll requested 2024-11-24T13:47:21,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041317 2024-11-24T13:47:21,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041309 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041317 2024-11-24T13:47:21,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,325 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,325 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=597, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,325 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041317) roll requested 2024-11-24T13:47:21,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041325 2024-11-24T13:47:21,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041317 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041325 2024-11-24T13:47:21,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,334 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,334 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=598, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,335 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041325) roll requested 2024-11-24T13:47:21,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041335 2024-11-24T13:47:21,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041325 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041335 2024-11-24T13:47:21,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,342 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,342 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=599, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,342 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041335) roll requested 2024-11-24T13:47:21,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041343 2024-11-24T13:47:21,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041335 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041343 2024-11-24T13:47:21,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,349 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,349 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=600, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,349 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041343) roll requested 2024-11-24T13:47:21,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041349 2024-11-24T13:47:21,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041343 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041349 2024-11-24T13:47:21,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,357 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,357 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=601, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,358 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041349) roll requested 2024-11-24T13:47:21,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041358 2024-11-24T13:47:21,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041349 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041358 2024-11-24T13:47:21,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,366 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,366 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=602, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,366 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041358) roll requested 2024-11-24T13:47:21,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041366 2024-11-24T13:47:21,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041358 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041366 2024-11-24T13:47:21,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,380 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,380 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=603, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,380 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041366) roll requested 2024-11-24T13:47:21,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041380 2024-11-24T13:47:21,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041366 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041380 2024-11-24T13:47:21,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,386 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,386 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=604, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,386 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041380) roll requested 2024-11-24T13:47:21,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041386 2024-11-24T13:47:21,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041380 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041386 2024-11-24T13:47:21,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,391 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,392 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=605, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,392 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041386) roll requested 2024-11-24T13:47:21,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041392 2024-11-24T13:47:21,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041386 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041392 2024-11-24T13:47:21,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,399 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,399 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=606, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,399 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041392) roll requested 2024-11-24T13:47:21,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041399 2024-11-24T13:47:21,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041392 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041399 2024-11-24T13:47:21,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,409 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,409 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=607, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,409 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041399) roll requested 2024-11-24T13:47:21,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041409 2024-11-24T13:47:21,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041399 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041409 2024-11-24T13:47:21,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,414 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,414 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=608, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,415 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041409) roll requested 2024-11-24T13:47:21,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041415 2024-11-24T13:47:21,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041409 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041415 2024-11-24T13:47:21,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,423 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,423 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=609, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,423 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041415) roll requested 2024-11-24T13:47:21,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041423 2024-11-24T13:47:21,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041415 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041423 2024-11-24T13:47:21,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,430 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,431 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=610, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,431 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041423) roll requested 2024-11-24T13:47:21,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041431 2024-11-24T13:47:21,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041423 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041431 2024-11-24T13:47:21,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,438 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,438 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=611, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,438 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041431) roll requested 2024-11-24T13:47:21,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041438 2024-11-24T13:47:21,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041431 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041438 2024-11-24T13:47:21,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,447 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,447 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=612, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,447 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041438) roll requested 2024-11-24T13:47:21,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041447 2024-11-24T13:47:21,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041438 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041447 2024-11-24T13:47:21,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,453 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,453 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=613, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,453 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041447) roll requested 2024-11-24T13:47:21,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041453 2024-11-24T13:47:21,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041447 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041453 2024-11-24T13:47:21,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,459 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,459 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=614, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,459 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041453) roll requested 2024-11-24T13:47:21,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041459 2024-11-24T13:47:21,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041453 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041459 2024-11-24T13:47:21,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,467 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,467 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=615, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,467 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041459) roll requested 2024-11-24T13:47:21,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041467 2024-11-24T13:47:21,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041459 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041467 2024-11-24T13:47:21,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,477 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,477 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=616, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,477 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041467) roll requested 2024-11-24T13:47:21,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041477 2024-11-24T13:47:21,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041467 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041477 2024-11-24T13:47:21,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,483 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,483 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=617, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,483 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041477) roll requested 2024-11-24T13:47:21,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041484 2024-11-24T13:47:21,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041477 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041484 2024-11-24T13:47:21,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,489 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,489 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=618, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,489 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041484) roll requested 2024-11-24T13:47:21,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041489 2024-11-24T13:47:21,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041484 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041489 2024-11-24T13:47:21,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,494 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,494 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=619, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,494 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041489) roll requested 2024-11-24T13:47:21,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041494 2024-11-24T13:47:21,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041489 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041494 2024-11-24T13:47:21,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,500 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,500 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=620, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,500 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041494) roll requested 2024-11-24T13:47:21,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041500 2024-11-24T13:47:21,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041494 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041500 2024-11-24T13:47:21,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,506 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,506 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=621, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,506 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041500) roll requested 2024-11-24T13:47:21,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041507 2024-11-24T13:47:21,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041500 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041507 2024-11-24T13:47:21,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,514 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,514 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=622, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,514 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041507) roll requested 2024-11-24T13:47:21,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041514 2024-11-24T13:47:21,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041507 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041514 2024-11-24T13:47:21,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,522 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,522 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=623, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,522 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041514) roll requested 2024-11-24T13:47:21,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041522 2024-11-24T13:47:21,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041514 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041522 2024-11-24T13:47:21,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,529 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,529 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=624, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,529 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041522) roll requested 2024-11-24T13:47:21,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041530 2024-11-24T13:47:21,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041522 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041530 2024-11-24T13:47:21,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,536 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,537 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=625, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,537 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041530) roll requested 2024-11-24T13:47:21,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041537 2024-11-24T13:47:21,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041530 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041537 2024-11-24T13:47:21,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,544 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,544 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=626, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,545 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041537) roll requested 2024-11-24T13:47:21,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041545 2024-11-24T13:47:21,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041537 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041545 2024-11-24T13:47:21,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,553 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,553 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=627, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,553 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041545) roll requested 2024-11-24T13:47:21,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041553 2024-11-24T13:47:21,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041545 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041553 2024-11-24T13:47:21,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,562 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,563 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=628, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,563 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041553) roll requested 2024-11-24T13:47:21,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041563 2024-11-24T13:47:21,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041553 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041563 2024-11-24T13:47:21,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,571 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,571 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=629, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,571 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041563) roll requested 2024-11-24T13:47:21,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041571 2024-11-24T13:47:21,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041563 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041571 2024-11-24T13:47:21,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,578 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,579 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=630, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,579 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041571) roll requested 2024-11-24T13:47:21,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041579 2024-11-24T13:47:21,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041571 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041579 2024-11-24T13:47:21,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,588 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,588 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=631, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,588 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041579) roll requested 2024-11-24T13:47:21,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041589 2024-11-24T13:47:21,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041579 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041589 2024-11-24T13:47:21,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,599 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,599 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=632, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,599 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041589) roll requested 2024-11-24T13:47:21,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041599 2024-11-24T13:47:21,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041589 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041599 2024-11-24T13:47:21,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,609 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,609 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=633, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,609 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041599) roll requested 2024-11-24T13:47:21,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041609 2024-11-24T13:47:21,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041599 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041609 2024-11-24T13:47:21,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,617 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,618 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=634, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,618 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041609) roll requested 2024-11-24T13:47:21,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041618 2024-11-24T13:47:21,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041609 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041618 2024-11-24T13:47:21,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,624 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,624 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=635, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,624 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041618) roll requested 2024-11-24T13:47:21,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041624 2024-11-24T13:47:21,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041618 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041624 2024-11-24T13:47:21,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,631 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,631 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=636, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,632 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041624) roll requested 2024-11-24T13:47:21,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041632 2024-11-24T13:47:21,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041624 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041632 2024-11-24T13:47:21,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,641 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,641 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=637, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,641 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041632) roll requested 2024-11-24T13:47:21,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041641 2024-11-24T13:47:21,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041632 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041641 2024-11-24T13:47:21,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,655 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,655 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=638, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,655 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041641) roll requested 2024-11-24T13:47:21,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041655 2024-11-24T13:47:21,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041641 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041655 2024-11-24T13:47:21,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,661 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,661 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=639, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,661 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041655) roll requested 2024-11-24T13:47:21,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041661 2024-11-24T13:47:21,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041655 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041661 2024-11-24T13:47:21,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,674 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,674 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=640, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,674 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041661) roll requested 2024-11-24T13:47:21,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041674 2024-11-24T13:47:21,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041661 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041674 2024-11-24T13:47:21,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,682 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,683 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=641, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,683 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041674) roll requested 2024-11-24T13:47:21,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041683 2024-11-24T13:47:21,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041674 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041683 2024-11-24T13:47:21,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,688 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,688 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=642, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,688 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041683) roll requested 2024-11-24T13:47:21,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041688 2024-11-24T13:47:21,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041683 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041688 2024-11-24T13:47:21,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,693 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,693 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=643, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,693 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041688) roll requested 2024-11-24T13:47:21,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041694 2024-11-24T13:47:21,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041688 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041694 2024-11-24T13:47:21,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,699 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,699 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=644, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,699 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041694) roll requested 2024-11-24T13:47:21,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041699 2024-11-24T13:47:21,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041694 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041699 2024-11-24T13:47:21,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,706 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,706 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=645, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,706 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041699) roll requested 2024-11-24T13:47:21,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041706 2024-11-24T13:47:21,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041699 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041706 2024-11-24T13:47:21,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,713 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,714 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=646, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,714 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041706) roll requested 2024-11-24T13:47:21,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041714 2024-11-24T13:47:21,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041706 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041714 2024-11-24T13:47:21,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,722 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,723 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=647, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,723 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041714) roll requested 2024-11-24T13:47:21,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041723 2024-11-24T13:47:21,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041714 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041723 2024-11-24T13:47:21,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,732 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,732 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=648, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,732 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041723) roll requested 2024-11-24T13:47:21,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041733 2024-11-24T13:47:21,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041723 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041733 2024-11-24T13:47:21,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,740 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,741 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=649, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,741 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041733) roll requested 2024-11-24T13:47:21,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041741 2024-11-24T13:47:21,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041733 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041741 2024-11-24T13:47:21,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,748 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,748 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=650, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,749 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041741) roll requested 2024-11-24T13:47:21,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041749 2024-11-24T13:47:21,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041741 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041749 2024-11-24T13:47:21,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,757 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,757 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=651, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,757 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041749) roll requested 2024-11-24T13:47:21,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041757 2024-11-24T13:47:21,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041749 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041757 2024-11-24T13:47:21,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,766 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,766 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=652, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,766 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041757) roll requested 2024-11-24T13:47:21,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041767 2024-11-24T13:47:21,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041757 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041767 2024-11-24T13:47:21,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,777 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,777 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=653, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,777 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041767) roll requested 2024-11-24T13:47:21,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041778 2024-11-24T13:47:21,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041767 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041778 2024-11-24T13:47:21,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,787 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,787 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=654, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,788 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041778) roll requested 2024-11-24T13:47:21,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041788 2024-11-24T13:47:21,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041778 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041788 2024-11-24T13:47:21,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,797 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,797 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=655, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,797 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041788) roll requested 2024-11-24T13:47:21,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041797 2024-11-24T13:47:21,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041788 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041797 2024-11-24T13:47:21,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,805 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,805 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=656, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,805 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041797) roll requested 2024-11-24T13:47:21,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041805 2024-11-24T13:47:21,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041797 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041805 2024-11-24T13:47:21,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,813 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,813 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=657, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,813 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041805) roll requested 2024-11-24T13:47:21,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041813 2024-11-24T13:47:21,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041805 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041813 2024-11-24T13:47:21,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,820 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,820 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=658, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,821 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041813) roll requested 2024-11-24T13:47:21,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041821 2024-11-24T13:47:21,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041813 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041821 2024-11-24T13:47:21,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,828 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,828 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=659, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,828 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041821) roll requested 2024-11-24T13:47:21,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041829 2024-11-24T13:47:21,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041821 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041829 2024-11-24T13:47:21,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,837 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,837 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=660, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,838 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041829) roll requested 2024-11-24T13:47:21,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041838 2024-11-24T13:47:21,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041829 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041838 2024-11-24T13:47:21,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,844 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,845 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=661, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,845 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041838) roll requested 2024-11-24T13:47:21,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041845 2024-11-24T13:47:21,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041838 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041845 2024-11-24T13:47:21,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,854 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,854 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=662, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,854 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041845) roll requested 2024-11-24T13:47:21,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041854 2024-11-24T13:47:21,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041845 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041854 2024-11-24T13:47:21,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,861 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,861 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=663, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,861 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041854) roll requested 2024-11-24T13:47:21,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041862 2024-11-24T13:47:21,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041854 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041862 2024-11-24T13:47:21,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,867 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,867 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=664, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,868 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041862) roll requested 2024-11-24T13:47:21,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041868 2024-11-24T13:47:21,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041862 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041868 2024-11-24T13:47:21,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,875 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,876 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=665, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,876 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041868) roll requested 2024-11-24T13:47:21,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041876 2024-11-24T13:47:21,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041868 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041876 2024-11-24T13:47:21,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,886 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,887 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=666, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,887 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041876) roll requested 2024-11-24T13:47:21,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041887 2024-11-24T13:47:21,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041876 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041887 2024-11-24T13:47:21,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,896 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,896 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=667, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,896 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041887) roll requested 2024-11-24T13:47:21,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041896 2024-11-24T13:47:21,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041887 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041896 2024-11-24T13:47:21,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,901 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,902 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=668, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,902 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041896) roll requested 2024-11-24T13:47:21,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041902 2024-11-24T13:47:21,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041896 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041902 2024-11-24T13:47:21,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,907 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,908 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=669, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,908 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041902) roll requested 2024-11-24T13:47:21,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041908 2024-11-24T13:47:21,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041902 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041908 2024-11-24T13:47:21,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,915 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,915 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=670, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,915 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041908) roll requested 2024-11-24T13:47:21,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041915 2024-11-24T13:47:21,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041908 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041915 2024-11-24T13:47:21,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,922 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,923 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=671, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,923 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041915) roll requested 2024-11-24T13:47:21,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041923 2024-11-24T13:47:21,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041915 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041923 2024-11-24T13:47:21,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,931 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,931 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=672, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,931 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041923) roll requested 2024-11-24T13:47:21,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041931 2024-11-24T13:47:21,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041923 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041931 2024-11-24T13:47:21,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,936 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,936 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=673, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,936 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041931) roll requested 2024-11-24T13:47:21,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041936 2024-11-24T13:47:21,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041931 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041936 2024-11-24T13:47:21,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,941 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,941 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=674, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,941 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041936) roll requested 2024-11-24T13:47:21,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041942 2024-11-24T13:47:21,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041936 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041942 2024-11-24T13:47:21,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,947 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,947 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=675, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,947 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041942) roll requested 2024-11-24T13:47:21,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041947 2024-11-24T13:47:21,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041942 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041947 2024-11-24T13:47:21,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,954 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,954 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=676, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,954 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041947) roll requested 2024-11-24T13:47:21,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041954 2024-11-24T13:47:21,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041947 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041954 2024-11-24T13:47:21,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,961 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,961 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=677, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,961 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041954) roll requested 2024-11-24T13:47:21,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041961 2024-11-24T13:47:21,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041954 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041961 2024-11-24T13:47:21,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,967 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,967 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=678, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,967 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041961) roll requested 2024-11-24T13:47:21,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041967 2024-11-24T13:47:21,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041961 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041967 2024-11-24T13:47:21,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,973 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,973 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=679, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,973 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041967) roll requested 2024-11-24T13:47:21,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041973 2024-11-24T13:47:21,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041967 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041973 2024-11-24T13:47:21,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,978 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,978 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=680, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,979 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041973) roll requested 2024-11-24T13:47:21,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041979 2024-11-24T13:47:21,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041973 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041979 2024-11-24T13:47:21,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,985 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,985 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=681, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,985 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041979) roll requested 2024-11-24T13:47:21,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041985 2024-11-24T13:47:21,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041979 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041985 2024-11-24T13:47:21,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,991 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,991 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=682, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,992 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041985) roll requested 2024-11-24T13:47:21,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041992 2024-11-24T13:47:21,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:21,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041985 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041992 2024-11-24T13:47:21,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:21,997 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:21,997 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=683, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:21,997 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:21,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041992) roll requested 2024-11-24T13:47:21,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456041997 2024-11-24T13:47:22,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041992 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041997 2024-11-24T13:47:22,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,002 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,002 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=684, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,002 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456041997) roll requested 2024-11-24T13:47:22,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042003 2024-11-24T13:47:22,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456041997 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042003 2024-11-24T13:47:22,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,010 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,010 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=685, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,010 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042003) roll requested 2024-11-24T13:47:22,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042010 2024-11-24T13:47:22,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042003 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042010 2024-11-24T13:47:22,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,015 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,016 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=686, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,016 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042010) roll requested 2024-11-24T13:47:22,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042016 2024-11-24T13:47:22,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042010 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042016 2024-11-24T13:47:22,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,021 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,021 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=687, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,021 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042016) roll requested 2024-11-24T13:47:22,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042021 2024-11-24T13:47:22,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042016 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042021 2024-11-24T13:47:22,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,027 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,027 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=688, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,027 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042021) roll requested 2024-11-24T13:47:22,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042027 2024-11-24T13:47:22,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042021 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042027 2024-11-24T13:47:22,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,032 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,032 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=689, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,032 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042027) roll requested 2024-11-24T13:47:22,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042032 2024-11-24T13:47:22,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042027 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042032 2024-11-24T13:47:22,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,038 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,038 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=690, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,038 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042032) roll requested 2024-11-24T13:47:22,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042038 2024-11-24T13:47:22,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042032 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042038 2024-11-24T13:47:22,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,045 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,045 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=691, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,045 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042038) roll requested 2024-11-24T13:47:22,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042045 2024-11-24T13:47:22,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042038 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042045 2024-11-24T13:47:22,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,051 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,051 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=692, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,051 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042045) roll requested 2024-11-24T13:47:22,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042051 2024-11-24T13:47:22,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042045 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042051 2024-11-24T13:47:22,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,056 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,056 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=693, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,056 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042051) roll requested 2024-11-24T13:47:22,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042056 2024-11-24T13:47:22,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042051 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042056 2024-11-24T13:47:22,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,062 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,062 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=694, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,062 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042056) roll requested 2024-11-24T13:47:22,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042062 2024-11-24T13:47:22,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042056 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042062 2024-11-24T13:47:22,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,067 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,068 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=695, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,068 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042062) roll requested 2024-11-24T13:47:22,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042068 2024-11-24T13:47:22,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042062 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042068 2024-11-24T13:47:22,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,073 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,073 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=696, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,073 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042068) roll requested 2024-11-24T13:47:22,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042073 2024-11-24T13:47:22,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042068 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042073 2024-11-24T13:47:22,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,080 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,080 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=697, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,080 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042073) roll requested 2024-11-24T13:47:22,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042080 2024-11-24T13:47:22,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042073 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042080 2024-11-24T13:47:22,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,086 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,086 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=698, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,086 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042080) roll requested 2024-11-24T13:47:22,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042086 2024-11-24T13:47:22,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042080 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042086 2024-11-24T13:47:22,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,093 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,093 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=699, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,093 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042086) roll requested 2024-11-24T13:47:22,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042093 2024-11-24T13:47:22,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042086 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042093 2024-11-24T13:47:22,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,099 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,099 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=700, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,099 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042093) roll requested 2024-11-24T13:47:22,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042099 2024-11-24T13:47:22,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042093 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042099 2024-11-24T13:47:22,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,105 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,105 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=701, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,105 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042099) roll requested 2024-11-24T13:47:22,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042105 2024-11-24T13:47:22,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042099 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042105 2024-11-24T13:47:22,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,111 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,111 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=702, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,111 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042105) roll requested 2024-11-24T13:47:22,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042111 2024-11-24T13:47:22,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042105 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042111 2024-11-24T13:47:22,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,116 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,117 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=703, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,117 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042111) roll requested 2024-11-24T13:47:22,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042117 2024-11-24T13:47:22,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042111 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042117 2024-11-24T13:47:22,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,122 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,122 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=704, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,122 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042117) roll requested 2024-11-24T13:47:22,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042122 2024-11-24T13:47:22,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042117 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042122 2024-11-24T13:47:22,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,128 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,128 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=705, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,128 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042122) roll requested 2024-11-24T13:47:22,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042128 2024-11-24T13:47:22,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042122 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042128 2024-11-24T13:47:22,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,133 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,133 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=706, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,133 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042128) roll requested 2024-11-24T13:47:22,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042133 2024-11-24T13:47:22,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042128 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042133 2024-11-24T13:47:22,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,138 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,138 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=707, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,138 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042133) roll requested 2024-11-24T13:47:22,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042138 2024-11-24T13:47:22,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042133 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042138 2024-11-24T13:47:22,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,144 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,144 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=708, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,144 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042138) roll requested 2024-11-24T13:47:22,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042144 2024-11-24T13:47:22,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042138 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042144 2024-11-24T13:47:22,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,149 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,150 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=709, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,150 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042144) roll requested 2024-11-24T13:47:22,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042150 2024-11-24T13:47:22,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042144 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042150 2024-11-24T13:47:22,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,155 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,155 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=710, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,156 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042150) roll requested 2024-11-24T13:47:22,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042156 2024-11-24T13:47:22,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042150 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042156 2024-11-24T13:47:22,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,162 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,162 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=711, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,162 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042156) roll requested 2024-11-24T13:47:22,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042162 2024-11-24T13:47:22,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042156 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042162 2024-11-24T13:47:22,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,168 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,168 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=712, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,168 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042162) roll requested 2024-11-24T13:47:22,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042168 2024-11-24T13:47:22,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042162 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042168 2024-11-24T13:47:22,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,174 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,174 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=713, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,174 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042168) roll requested 2024-11-24T13:47:22,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042175 2024-11-24T13:47:22,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042168 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042175 2024-11-24T13:47:22,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,181 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,181 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=714, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,181 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042175) roll requested 2024-11-24T13:47:22,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042181 2024-11-24T13:47:22,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042175 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042181 2024-11-24T13:47:22,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,187 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,187 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=715, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,187 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042181) roll requested 2024-11-24T13:47:22,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042187 2024-11-24T13:47:22,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042181 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042187 2024-11-24T13:47:22,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,194 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,194 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=716, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,194 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042187) roll requested 2024-11-24T13:47:22,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042194 2024-11-24T13:47:22,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042187 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042194 2024-11-24T13:47:22,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,200 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,201 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=717, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,201 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042194) roll requested 2024-11-24T13:47:22,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042201 2024-11-24T13:47:22,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042194 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042201 2024-11-24T13:47:22,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,206 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,207 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=718, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,207 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042201) roll requested 2024-11-24T13:47:22,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042207 2024-11-24T13:47:22,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042201 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042207 2024-11-24T13:47:22,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,212 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,212 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=719, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,212 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042207) roll requested 2024-11-24T13:47:22,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042212 2024-11-24T13:47:22,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042207 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042212 2024-11-24T13:47:22,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,217 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,218 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=720, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,218 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042212) roll requested 2024-11-24T13:47:22,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042218 2024-11-24T13:47:22,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042212 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042218 2024-11-24T13:47:22,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,224 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,224 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=721, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,224 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042218) roll requested 2024-11-24T13:47:22,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042224 2024-11-24T13:47:22,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042218 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042224 2024-11-24T13:47:22,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,231 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,231 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=722, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,231 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042224) roll requested 2024-11-24T13:47:22,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042231 2024-11-24T13:47:22,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042224 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042231 2024-11-24T13:47:22,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,239 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,239 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=723, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,239 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042231) roll requested 2024-11-24T13:47:22,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042239 2024-11-24T13:47:22,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042231 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042239 2024-11-24T13:47:22,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,246 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,246 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=724, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,246 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042239) roll requested 2024-11-24T13:47:22,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042246 2024-11-24T13:47:22,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042239 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042246 2024-11-24T13:47:22,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,253 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,253 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=725, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,253 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042246) roll requested 2024-11-24T13:47:22,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042253 2024-11-24T13:47:22,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042246 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042253 2024-11-24T13:47:22,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,261 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,261 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=726, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,261 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042253) roll requested 2024-11-24T13:47:22,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042261 2024-11-24T13:47:22,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042253 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042261 2024-11-24T13:47:22,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,270 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,271 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=727, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,271 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042261) roll requested 2024-11-24T13:47:22,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042271 2024-11-24T13:47:22,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042261 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042271 2024-11-24T13:47:22,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,278 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,278 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=728, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,278 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042271) roll requested 2024-11-24T13:47:22,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042278 2024-11-24T13:47:22,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042271 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042278 2024-11-24T13:47:22,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,287 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,288 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=729, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,288 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042278) roll requested 2024-11-24T13:47:22,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042288 2024-11-24T13:47:22,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042278 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042288 2024-11-24T13:47:22,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,296 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,296 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=730, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,296 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042288) roll requested 2024-11-24T13:47:22,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042296 2024-11-24T13:47:22,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042288 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042296 2024-11-24T13:47:22,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,302 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,303 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=731, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,303 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042296) roll requested 2024-11-24T13:47:22,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042303 2024-11-24T13:47:22,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042296 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042303 2024-11-24T13:47:22,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,312 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,312 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=732, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,312 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042303) roll requested 2024-11-24T13:47:22,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042312 2024-11-24T13:47:22,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042303 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042312 2024-11-24T13:47:22,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,323 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,323 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=733, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,323 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042312) roll requested 2024-11-24T13:47:22,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042323 2024-11-24T13:47:22,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042312 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042323 2024-11-24T13:47:22,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,334 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,334 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=734, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,335 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042323) roll requested 2024-11-24T13:47:22,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042335 2024-11-24T13:47:22,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042323 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042335 2024-11-24T13:47:22,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,340 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,340 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=735, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,340 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042335) roll requested 2024-11-24T13:47:22,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042340 2024-11-24T13:47:22,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042335 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042340 2024-11-24T13:47:22,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,349 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,349 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=736, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,350 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042340) roll requested 2024-11-24T13:47:22,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042350 2024-11-24T13:47:22,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042340 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042350 2024-11-24T13:47:22,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,355 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,355 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=737, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,355 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042350) roll requested 2024-11-24T13:47:22,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042355 2024-11-24T13:47:22,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042350 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042355 2024-11-24T13:47:22,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,365 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,365 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=738, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,365 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042355) roll requested 2024-11-24T13:47:22,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042365 2024-11-24T13:47:22,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042355 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042365 2024-11-24T13:47:22,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,377 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,377 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=739, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,377 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042365) roll requested 2024-11-24T13:47:22,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042378 2024-11-24T13:47:22,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042365 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042378 2024-11-24T13:47:22,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,387 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,387 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=740, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,387 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042378) roll requested 2024-11-24T13:47:22,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042387 2024-11-24T13:47:22,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042378 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042387 2024-11-24T13:47:22,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,394 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,394 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=741, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,394 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042387) roll requested 2024-11-24T13:47:22,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042394 2024-11-24T13:47:22,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042387 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042394 2024-11-24T13:47:22,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,399 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,400 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=742, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,400 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042394) roll requested 2024-11-24T13:47:22,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042400 2024-11-24T13:47:22,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042394 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042400 2024-11-24T13:47:22,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,411 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,411 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=743, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,411 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042400) roll requested 2024-11-24T13:47:22,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042411 2024-11-24T13:47:22,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042400 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042411 2024-11-24T13:47:22,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,420 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,420 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=744, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,420 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042411) roll requested 2024-11-24T13:47:22,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042420 2024-11-24T13:47:22,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042411 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042420 2024-11-24T13:47:22,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,431 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,431 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=745, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,431 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042420) roll requested 2024-11-24T13:47:22,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042431 2024-11-24T13:47:22,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042420 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042431 2024-11-24T13:47:22,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,437 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,437 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=746, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,437 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042431) roll requested 2024-11-24T13:47:22,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042437 2024-11-24T13:47:22,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042431 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042437 2024-11-24T13:47:22,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,443 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,443 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=747, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,443 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042437) roll requested 2024-11-24T13:47:22,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042443 2024-11-24T13:47:22,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042437 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042443 2024-11-24T13:47:22,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,448 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,449 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=748, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,449 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042443) roll requested 2024-11-24T13:47:22,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042449 2024-11-24T13:47:22,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042443 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042449 2024-11-24T13:47:22,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,456 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,456 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=749, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,456 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042449) roll requested 2024-11-24T13:47:22,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042456 2024-11-24T13:47:22,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042449 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042456 2024-11-24T13:47:22,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,462 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,462 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=750, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,462 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042456) roll requested 2024-11-24T13:47:22,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042462 2024-11-24T13:47:22,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042456 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042462 2024-11-24T13:47:22,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,467 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,467 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=751, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,467 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042462) roll requested 2024-11-24T13:47:22,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042468 2024-11-24T13:47:22,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042462 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042468 2024-11-24T13:47:22,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,473 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,473 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=752, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,473 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042468) roll requested 2024-11-24T13:47:22,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042473 2024-11-24T13:47:22,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042468 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042473 2024-11-24T13:47:22,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,479 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,479 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=753, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,479 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042473) roll requested 2024-11-24T13:47:22,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042479 2024-11-24T13:47:22,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042473 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042479 2024-11-24T13:47:22,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,485 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,485 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=754, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,485 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042479) roll requested 2024-11-24T13:47:22,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042485 2024-11-24T13:47:22,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042479 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042485 2024-11-24T13:47:22,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,489 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,490 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=755, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,490 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042485) roll requested 2024-11-24T13:47:22,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042490 2024-11-24T13:47:22,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042490 2024-11-24T13:47:22,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,495 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,495 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=756, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,495 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042490) roll requested 2024-11-24T13:47:22,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042495 2024-11-24T13:47:22,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042490 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042495 2024-11-24T13:47:22,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,501 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,501 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=757, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,501 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042495) roll requested 2024-11-24T13:47:22,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042502 2024-11-24T13:47:22,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042495 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042502 2024-11-24T13:47:22,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,507 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,508 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=758, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,508 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042502) roll requested 2024-11-24T13:47:22,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042508 2024-11-24T13:47:22,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042502 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042508 2024-11-24T13:47:22,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,515 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,515 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=759, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,515 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042508) roll requested 2024-11-24T13:47:22,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042515 2024-11-24T13:47:22,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042508 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042515 2024-11-24T13:47:22,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,522 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,523 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=760, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,523 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042515) roll requested 2024-11-24T13:47:22,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042523 2024-11-24T13:47:22,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042515 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042523 2024-11-24T13:47:22,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,532 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,532 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=761, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,532 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042523) roll requested 2024-11-24T13:47:22,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042533 2024-11-24T13:47:22,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042523 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042533 2024-11-24T13:47:22,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,541 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,542 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=762, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,542 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042533) roll requested 2024-11-24T13:47:22,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042542 2024-11-24T13:47:22,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042533 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042542 2024-11-24T13:47:22,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,549 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,550 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=763, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,550 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042542) roll requested 2024-11-24T13:47:22,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042550 2024-11-24T13:47:22,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042542 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042550 2024-11-24T13:47:22,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,558 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,558 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=764, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,559 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042550) roll requested 2024-11-24T13:47:22,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042559 2024-11-24T13:47:22,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042550 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042559 2024-11-24T13:47:22,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,566 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,566 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=765, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,566 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042559) roll requested 2024-11-24T13:47:22,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042566 2024-11-24T13:47:22,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042559 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042566 2024-11-24T13:47:22,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,575 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,575 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=766, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,575 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042566) roll requested 2024-11-24T13:47:22,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042575 2024-11-24T13:47:22,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042566 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042575 2024-11-24T13:47:22,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,584 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,584 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=767, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,584 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042575) roll requested 2024-11-24T13:47:22,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042584 2024-11-24T13:47:22,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042575 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042584 2024-11-24T13:47:22,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,590 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,590 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=768, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,590 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042584) roll requested 2024-11-24T13:47:22,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042591 2024-11-24T13:47:22,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042584 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042591 2024-11-24T13:47:22,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,599 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,599 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=769, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,599 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042591) roll requested 2024-11-24T13:47:22,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042599 2024-11-24T13:47:22,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042591 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042599 2024-11-24T13:47:22,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,607 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,607 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=770, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,607 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042599) roll requested 2024-11-24T13:47:22,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042608 2024-11-24T13:47:22,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042599 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042608 2024-11-24T13:47:22,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,615 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,615 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=771, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,615 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042608) roll requested 2024-11-24T13:47:22,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042615 2024-11-24T13:47:22,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042608 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042615 2024-11-24T13:47:22,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,623 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,623 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=772, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,623 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042615) roll requested 2024-11-24T13:47:22,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042623 2024-11-24T13:47:22,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042615 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042623 2024-11-24T13:47:22,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,631 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,631 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=773, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,631 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042623) roll requested 2024-11-24T13:47:22,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042631 2024-11-24T13:47:22,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042623 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042631 2024-11-24T13:47:22,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,638 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,638 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=774, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,638 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042631) roll requested 2024-11-24T13:47:22,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042638 2024-11-24T13:47:22,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042631 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042638 2024-11-24T13:47:22,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,646 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,646 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=775, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,646 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042638) roll requested 2024-11-24T13:47:22,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042647 2024-11-24T13:47:22,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042638 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042647 2024-11-24T13:47:22,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,655 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,655 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=776, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,655 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042647) roll requested 2024-11-24T13:47:22,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042655 2024-11-24T13:47:22,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042647 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042655 2024-11-24T13:47:22,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,662 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,662 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=777, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,662 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042655) roll requested 2024-11-24T13:47:22,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042662 2024-11-24T13:47:22,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042655 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042662 2024-11-24T13:47:22,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,670 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,670 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=778, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,671 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042662) roll requested 2024-11-24T13:47:22,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042671 2024-11-24T13:47:22,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042662 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042671 2024-11-24T13:47:22,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,678 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,678 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=779, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,678 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042671) roll requested 2024-11-24T13:47:22,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042678 2024-11-24T13:47:22,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042671 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042678 2024-11-24T13:47:22,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,686 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,687 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=780, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,687 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042678) roll requested 2024-11-24T13:47:22,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042687 2024-11-24T13:47:22,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042678 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042687 2024-11-24T13:47:22,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,694 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,694 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=781, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,694 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042687) roll requested 2024-11-24T13:47:22,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042695 2024-11-24T13:47:22,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042687 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042695 2024-11-24T13:47:22,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,701 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,701 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=782, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,701 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042695) roll requested 2024-11-24T13:47:22,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042701 2024-11-24T13:47:22,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042695 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042701 2024-11-24T13:47:22,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,709 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=783, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042701) roll requested 2024-11-24T13:47:22,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042709 2024-11-24T13:47:22,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042709 2024-11-24T13:47:22,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,717 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,717 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=784, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,717 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042709) roll requested 2024-11-24T13:47:22,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042718 2024-11-24T13:47:22,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042718 2024-11-24T13:47:22,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,726 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,726 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=785, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,726 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042718) roll requested 2024-11-24T13:47:22,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042726 2024-11-24T13:47:22,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042718 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042726 2024-11-24T13:47:22,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,734 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,734 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=786, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,734 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042726) roll requested 2024-11-24T13:47:22,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042734 2024-11-24T13:47:22,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042726 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042734 2024-11-24T13:47:22,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,740 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,740 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=787, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,740 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042734) roll requested 2024-11-24T13:47:22,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042740 2024-11-24T13:47:22,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042734 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042740 2024-11-24T13:47:22,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,747 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,747 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=788, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,747 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042740) roll requested 2024-11-24T13:47:22,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042747 2024-11-24T13:47:22,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042740 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042747 2024-11-24T13:47:22,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,755 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,755 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=789, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,755 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042747) roll requested 2024-11-24T13:47:22,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042755 2024-11-24T13:47:22,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042747 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042755 2024-11-24T13:47:22,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,765 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,766 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=790, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,766 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042755) roll requested 2024-11-24T13:47:22,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042766 2024-11-24T13:47:22,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042755 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042766 2024-11-24T13:47:22,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,775 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,776 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=791, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,776 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042766) roll requested 2024-11-24T13:47:22,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042776 2024-11-24T13:47:22,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042766 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042776 2024-11-24T13:47:22,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,782 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,783 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=792, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,783 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042776) roll requested 2024-11-24T13:47:22,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042783 2024-11-24T13:47:22,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042776 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042783 2024-11-24T13:47:22,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,789 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,789 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=793, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,789 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042783) roll requested 2024-11-24T13:47:22,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042789 2024-11-24T13:47:22,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042783 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042789 2024-11-24T13:47:22,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,794 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,795 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=794, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,795 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042789) roll requested 2024-11-24T13:47:22,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042795 2024-11-24T13:47:22,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042789 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042795 2024-11-24T13:47:22,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,799 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,800 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=795, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,800 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042795) roll requested 2024-11-24T13:47:22,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042800 2024-11-24T13:47:22,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042795 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042800 2024-11-24T13:47:22,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,806 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,807 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=796, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,807 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042800) roll requested 2024-11-24T13:47:22,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042807 2024-11-24T13:47:22,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042800 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042807 2024-11-24T13:47:22,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,814 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,814 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=797, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,814 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042807) roll requested 2024-11-24T13:47:22,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042814 2024-11-24T13:47:22,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042807 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042814 2024-11-24T13:47:22,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,822 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,822 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=798, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,822 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042814) roll requested 2024-11-24T13:47:22,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042822 2024-11-24T13:47:22,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042814 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042822 2024-11-24T13:47:22,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,830 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,831 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=799, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,831 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042822) roll requested 2024-11-24T13:47:22,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042831 2024-11-24T13:47:22,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042822 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042831 2024-11-24T13:47:22,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,839 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,839 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=800, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,839 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042831) roll requested 2024-11-24T13:47:22,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042839 2024-11-24T13:47:22,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042831 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042839 2024-11-24T13:47:22,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,847 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,847 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=801, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,848 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042839) roll requested 2024-11-24T13:47:22,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042848 2024-11-24T13:47:22,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042839 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042848 2024-11-24T13:47:22,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,855 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,856 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=802, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,856 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042848) roll requested 2024-11-24T13:47:22,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042856 2024-11-24T13:47:22,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042848 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042856 2024-11-24T13:47:22,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,865 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,865 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=803, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,865 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042856) roll requested 2024-11-24T13:47:22,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042865 2024-11-24T13:47:22,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042856 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042865 2024-11-24T13:47:22,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,874 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,874 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=804, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,875 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042865) roll requested 2024-11-24T13:47:22,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042875 2024-11-24T13:47:22,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042875 2024-11-24T13:47:22,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,883 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,884 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=805, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,884 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042875) roll requested 2024-11-24T13:47:22,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042884 2024-11-24T13:47:22,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042875 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042884 2024-11-24T13:47:22,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,890 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,890 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=806, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,890 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042884) roll requested 2024-11-24T13:47:22,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042890 2024-11-24T13:47:22,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042884 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042890 2024-11-24T13:47:22,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,897 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,897 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=807, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,897 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042890) roll requested 2024-11-24T13:47:22,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042897 2024-11-24T13:47:22,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042890 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042897 2024-11-24T13:47:22,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,906 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,906 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=808, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,906 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042897) roll requested 2024-11-24T13:47:22,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042906 2024-11-24T13:47:22,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042897 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042906 2024-11-24T13:47:22,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,914 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,915 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=809, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,915 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042906) roll requested 2024-11-24T13:47:22,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042915 2024-11-24T13:47:22,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042906 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042915 2024-11-24T13:47:22,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,924 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,925 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=810, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,925 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042915) roll requested 2024-11-24T13:47:22,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042925 2024-11-24T13:47:22,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042915 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042925 2024-11-24T13:47:22,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,932 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,932 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=811, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,933 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042925) roll requested 2024-11-24T13:47:22,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042933 2024-11-24T13:47:22,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042925 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042933 2024-11-24T13:47:22,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,938 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,938 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=812, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,938 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042933) roll requested 2024-11-24T13:47:22,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042938 2024-11-24T13:47:22,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042933 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042938 2024-11-24T13:47:22,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,945 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,945 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=813, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,945 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042938) roll requested 2024-11-24T13:47:22,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042945 2024-11-24T13:47:22,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042938 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042945 2024-11-24T13:47:22,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,952 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=814, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,952 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042945) roll requested 2024-11-24T13:47:22,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042953 2024-11-24T13:47:22,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042945 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042953 2024-11-24T13:47:22,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,960 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,960 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=815, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,961 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042953) roll requested 2024-11-24T13:47:22,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042961 2024-11-24T13:47:22,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042953 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042961 2024-11-24T13:47:22,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,969 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,970 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=816, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,970 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042961) roll requested 2024-11-24T13:47:22,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042970 2024-11-24T13:47:22,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042961 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042970 2024-11-24T13:47:22,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,978 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,978 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=817, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,979 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042970) roll requested 2024-11-24T13:47:22,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042979 2024-11-24T13:47:22,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042970 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042979 2024-11-24T13:47:22,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,986 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,986 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=818, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,987 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042979) roll requested 2024-11-24T13:47:22,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042987 2024-11-24T13:47:22,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:22,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042979 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042987 2024-11-24T13:47:22,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:22,995 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:22,995 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=819, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:22,996 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:22,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042987) roll requested 2024-11-24T13:47:22,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456042996 2024-11-24T13:47:23,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042987 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042996 2024-11-24T13:47:23,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,004 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,004 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=820, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,004 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456042996) roll requested 2024-11-24T13:47:23,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043004 2024-11-24T13:47:23,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456042996 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043004 2024-11-24T13:47:23,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,009 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,009 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=821, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,009 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043004) roll requested 2024-11-24T13:47:23,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043009 2024-11-24T13:47:23,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043004 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043009 2024-11-24T13:47:23,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,015 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,015 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=822, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,015 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043009) roll requested 2024-11-24T13:47:23,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043015 2024-11-24T13:47:23,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043009 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043015 2024-11-24T13:47:23,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,022 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,023 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=823, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,023 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043015) roll requested 2024-11-24T13:47:23,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043023 2024-11-24T13:47:23,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043015 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043023 2024-11-24T13:47:23,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,034 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,034 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=824, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,034 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043023) roll requested 2024-11-24T13:47:23,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043034 2024-11-24T13:47:23,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043023 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043034 2024-11-24T13:47:23,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,041 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,041 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=825, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,041 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043034) roll requested 2024-11-24T13:47:23,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043041 2024-11-24T13:47:23,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043034 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043041 2024-11-24T13:47:23,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,047 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,047 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=826, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,047 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043041) roll requested 2024-11-24T13:47:23,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043047 2024-11-24T13:47:23,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043041 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043047 2024-11-24T13:47:23,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,054 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,054 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=827, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,054 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043047) roll requested 2024-11-24T13:47:23,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043054 2024-11-24T13:47:23,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043047 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043054 2024-11-24T13:47:23,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,060 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,060 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=828, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,060 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043054) roll requested 2024-11-24T13:47:23,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043061 2024-11-24T13:47:23,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043054 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043061 2024-11-24T13:47:23,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,066 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,066 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=829, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,067 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043061) roll requested 2024-11-24T13:47:23,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043067 2024-11-24T13:47:23,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043061 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043067 2024-11-24T13:47:23,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,072 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,072 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=830, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,072 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043067) roll requested 2024-11-24T13:47:23,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043072 2024-11-24T13:47:23,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043067 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043072 2024-11-24T13:47:23,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,078 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,078 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=831, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,078 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043072) roll requested 2024-11-24T13:47:23,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043078 2024-11-24T13:47:23,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043072 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043078 2024-11-24T13:47:23,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,086 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,086 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=832, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,086 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043078) roll requested 2024-11-24T13:47:23,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043086 2024-11-24T13:47:23,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043078 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043086 2024-11-24T13:47:23,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,092 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,092 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=833, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,092 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043086) roll requested 2024-11-24T13:47:23,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043092 2024-11-24T13:47:23,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043086 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043092 2024-11-24T13:47:23,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,098 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,098 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=834, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,098 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043092) roll requested 2024-11-24T13:47:23,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043098 2024-11-24T13:47:23,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043092 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043098 2024-11-24T13:47:23,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,105 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=835, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,105 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043098) roll requested 2024-11-24T13:47:23,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043105 2024-11-24T13:47:23,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043098 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043105 2024-11-24T13:47:23,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,110 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,110 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=836, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,110 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043105) roll requested 2024-11-24T13:47:23,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043110 2024-11-24T13:47:23,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043105 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043110 2024-11-24T13:47:23,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,115 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,116 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=837, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,116 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043110) roll requested 2024-11-24T13:47:23,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043116 2024-11-24T13:47:23,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043110 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043116 2024-11-24T13:47:23,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,121 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,122 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=838, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,122 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043116) roll requested 2024-11-24T13:47:23,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043122 2024-11-24T13:47:23,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043116 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043122 2024-11-24T13:47:23,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,127 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,127 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=839, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,127 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043122) roll requested 2024-11-24T13:47:23,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043127 2024-11-24T13:47:23,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043122 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043127 2024-11-24T13:47:23,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,133 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,134 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=840, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,134 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043127) roll requested 2024-11-24T13:47:23,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043134 2024-11-24T13:47:23,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043127 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043134 2024-11-24T13:47:23,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,139 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,140 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=841, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,140 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043134) roll requested 2024-11-24T13:47:23,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043140 2024-11-24T13:47:23,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043134 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043140 2024-11-24T13:47:23,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,144 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,145 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=842, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,145 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043140) roll requested 2024-11-24T13:47:23,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043145 2024-11-24T13:47:23,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043140 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043145 2024-11-24T13:47:23,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,150 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,150 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=843, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,150 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043145) roll requested 2024-11-24T13:47:23,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043150 2024-11-24T13:47:23,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043145 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043150 2024-11-24T13:47:23,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,155 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,155 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=844, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,155 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043150) roll requested 2024-11-24T13:47:23,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043155 2024-11-24T13:47:23,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043150 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043155 2024-11-24T13:47:23,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,161 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,162 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=845, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,162 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043155) roll requested 2024-11-24T13:47:23,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043162 2024-11-24T13:47:23,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043155 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043162 2024-11-24T13:47:23,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,170 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,170 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=846, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,171 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043162) roll requested 2024-11-24T13:47:23,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043171 2024-11-24T13:47:23,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043162 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043171 2024-11-24T13:47:23,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,178 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,179 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=847, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,179 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043171) roll requested 2024-11-24T13:47:23,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043179 2024-11-24T13:47:23,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043171 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043179 2024-11-24T13:47:23,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,187 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,188 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=848, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,188 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043179) roll requested 2024-11-24T13:47:23,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043188 2024-11-24T13:47:23,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043179 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043188 2024-11-24T13:47:23,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,196 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,197 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=849, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,197 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043188) roll requested 2024-11-24T13:47:23,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043197 2024-11-24T13:47:23,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043188 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043197 2024-11-24T13:47:23,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,206 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,207 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=850, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,207 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043197) roll requested 2024-11-24T13:47:23,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043207 2024-11-24T13:47:23,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043197 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043207 2024-11-24T13:47:23,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,216 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,217 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=851, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,217 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043207) roll requested 2024-11-24T13:47:23,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043217 2024-11-24T13:47:23,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043207 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043217 2024-11-24T13:47:23,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,226 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,226 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=852, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,226 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043217) roll requested 2024-11-24T13:47:23,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043226 2024-11-24T13:47:23,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043217 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043226 2024-11-24T13:47:23,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,233 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,233 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=853, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,233 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043226) roll requested 2024-11-24T13:47:23,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043234 2024-11-24T13:47:23,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043226 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043234 2024-11-24T13:47:23,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,241 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,241 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=854, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,242 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043234) roll requested 2024-11-24T13:47:23,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043242 2024-11-24T13:47:23,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043234 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043242 2024-11-24T13:47:23,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,249 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,250 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=855, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,250 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043242) roll requested 2024-11-24T13:47:23,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043250 2024-11-24T13:47:23,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043242 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043250 2024-11-24T13:47:23,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,261 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,261 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=856, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,261 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043250) roll requested 2024-11-24T13:47:23,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043261 2024-11-24T13:47:23,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043250 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043261 2024-11-24T13:47:23,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,270 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,270 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=857, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,270 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043261) roll requested 2024-11-24T13:47:23,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043270 2024-11-24T13:47:23,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043261 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043270 2024-11-24T13:47:23,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,277 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,277 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=858, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,277 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043270) roll requested 2024-11-24T13:47:23,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043277 2024-11-24T13:47:23,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043270 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043277 2024-11-24T13:47:23,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,284 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,284 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=859, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,284 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043277) roll requested 2024-11-24T13:47:23,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043284 2024-11-24T13:47:23,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043284 2024-11-24T13:47:23,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,291 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,291 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=860, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,291 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043284) roll requested 2024-11-24T13:47:23,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043291 2024-11-24T13:47:23,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043284 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043291 2024-11-24T13:47:23,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,299 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,299 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=861, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,300 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043291) roll requested 2024-11-24T13:47:23,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043300 2024-11-24T13:47:23,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043291 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043300 2024-11-24T13:47:23,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,310 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,310 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=862, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,310 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043300) roll requested 2024-11-24T13:47:23,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043311 2024-11-24T13:47:23,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043300 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043311 2024-11-24T13:47:23,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,320 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,320 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=863, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,320 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043311) roll requested 2024-11-24T13:47:23,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043320 2024-11-24T13:47:23,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043311 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043320 2024-11-24T13:47:23,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,328 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,328 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=864, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,328 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043320) roll requested 2024-11-24T13:47:23,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043328 2024-11-24T13:47:23,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043320 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043328 2024-11-24T13:47:23,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,334 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,335 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=865, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,335 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043328) roll requested 2024-11-24T13:47:23,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043335 2024-11-24T13:47:23,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043328 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043335 2024-11-24T13:47:23,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,341 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,342 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=866, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,342 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043335) roll requested 2024-11-24T13:47:23,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043342 2024-11-24T13:47:23,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043335 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043342 2024-11-24T13:47:23,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,349 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,350 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=867, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,350 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043342) roll requested 2024-11-24T13:47:23,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043350 2024-11-24T13:47:23,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043342 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043350 2024-11-24T13:47:23,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,358 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,358 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=868, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,358 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043350) roll requested 2024-11-24T13:47:23,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043358 2024-11-24T13:47:23,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043350 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043358 2024-11-24T13:47:23,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,365 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,365 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=869, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,365 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043358) roll requested 2024-11-24T13:47:23,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043365 2024-11-24T13:47:23,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043358 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043365 2024-11-24T13:47:23,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,374 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,374 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=870, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,374 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043365) roll requested 2024-11-24T13:47:23,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043375 2024-11-24T13:47:23,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043365 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043375 2024-11-24T13:47:23,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,383 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,383 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=871, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,383 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043375) roll requested 2024-11-24T13:47:23,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043383 2024-11-24T13:47:23,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043375 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043383 2024-11-24T13:47:23,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,390 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,390 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=872, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,391 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043383) roll requested 2024-11-24T13:47:23,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043391 2024-11-24T13:47:23,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043383 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043391 2024-11-24T13:47:23,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,398 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,399 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=873, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,399 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043391) roll requested 2024-11-24T13:47:23,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043399 2024-11-24T13:47:23,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043391 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043399 2024-11-24T13:47:23,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,407 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,408 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=874, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,408 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043399) roll requested 2024-11-24T13:47:23,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043408 2024-11-24T13:47:23,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043399 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043408 2024-11-24T13:47:23,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,415 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,415 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=875, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,415 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043408) roll requested 2024-11-24T13:47:23,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043415 2024-11-24T13:47:23,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043408 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043415 2024-11-24T13:47:23,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,422 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,423 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=876, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,423 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043415) roll requested 2024-11-24T13:47:23,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043423 2024-11-24T13:47:23,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043415 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043423 2024-11-24T13:47:23,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,431 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,431 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=877, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,431 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043423) roll requested 2024-11-24T13:47:23,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043431 2024-11-24T13:47:23,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043423 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043431 2024-11-24T13:47:23,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,438 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,438 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=878, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,438 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043431) roll requested 2024-11-24T13:47:23,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043438 2024-11-24T13:47:23,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043431 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043438 2024-11-24T13:47:23,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,446 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,446 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=879, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,446 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043438) roll requested 2024-11-24T13:47:23,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043446 2024-11-24T13:47:23,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043438 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043446 2024-11-24T13:47:23,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,454 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,455 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=880, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,455 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043446) roll requested 2024-11-24T13:47:23,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043455 2024-11-24T13:47:23,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043446 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043455 2024-11-24T13:47:23,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,464 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,464 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=881, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,465 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043455) roll requested 2024-11-24T13:47:23,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043465 2024-11-24T13:47:23,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043455 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043465 2024-11-24T13:47:23,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,473 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,473 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=882, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,473 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043465) roll requested 2024-11-24T13:47:23,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043474 2024-11-24T13:47:23,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043465 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043474 2024-11-24T13:47:23,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,481 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,481 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=883, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,481 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043474) roll requested 2024-11-24T13:47:23,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043482 2024-11-24T13:47:23,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043474 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043482 2024-11-24T13:47:23,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,488 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,489 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=884, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,489 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043482) roll requested 2024-11-24T13:47:23,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043489 2024-11-24T13:47:23,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043482 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043489 2024-11-24T13:47:23,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,493 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,493 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=885, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,494 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043489) roll requested 2024-11-24T13:47:23,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043494 2024-11-24T13:47:23,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043489 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043494 2024-11-24T13:47:23,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,499 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,499 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=886, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,499 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043494) roll requested 2024-11-24T13:47:23,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043499 2024-11-24T13:47:23,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043494 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043499 2024-11-24T13:47:23,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,505 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,505 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=887, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,505 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043499) roll requested 2024-11-24T13:47:23,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043506 2024-11-24T13:47:23,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043499 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043506 2024-11-24T13:47:23,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,512 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,512 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=888, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,512 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043506) roll requested 2024-11-24T13:47:23,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043513 2024-11-24T13:47:23,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043506 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043513 2024-11-24T13:47:23,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,520 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,520 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=889, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,520 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043513) roll requested 2024-11-24T13:47:23,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043521 2024-11-24T13:47:23,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043513 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043521 2024-11-24T13:47:23,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,528 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,528 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=890, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,528 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043521) roll requested 2024-11-24T13:47:23,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043528 2024-11-24T13:47:23,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043521 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043528 2024-11-24T13:47:23,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,537 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,537 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=891, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,537 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043528) roll requested 2024-11-24T13:47:23,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043537 2024-11-24T13:47:23,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043528 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043537 2024-11-24T13:47:23,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,546 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,547 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=892, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,547 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043537) roll requested 2024-11-24T13:47:23,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043547 2024-11-24T13:47:23,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043537 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043547 2024-11-24T13:47:23,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,554 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,554 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=893, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,555 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043547) roll requested 2024-11-24T13:47:23,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043555 2024-11-24T13:47:23,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043547 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043555 2024-11-24T13:47:23,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,562 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,562 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=894, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,563 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043555) roll requested 2024-11-24T13:47:23,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043563 2024-11-24T13:47:23,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043555 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043563 2024-11-24T13:47:23,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,571 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,572 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=895, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,572 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043563) roll requested 2024-11-24T13:47:23,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043572 2024-11-24T13:47:23,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043563 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043572 2024-11-24T13:47:23,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,579 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,580 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=896, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,580 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043572) roll requested 2024-11-24T13:47:23,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043580 2024-11-24T13:47:23,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043572 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043580 2024-11-24T13:47:23,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,587 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,587 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=897, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,587 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043580) roll requested 2024-11-24T13:47:23,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043587 2024-11-24T13:47:23,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043580 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043587 2024-11-24T13:47:23,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,593 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,594 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=898, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,594 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043587) roll requested 2024-11-24T13:47:23,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043594 2024-11-24T13:47:23,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043587 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043594 2024-11-24T13:47:23,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,601 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,601 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=899, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,601 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043594) roll requested 2024-11-24T13:47:23,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043601 2024-11-24T13:47:23,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043594 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043601 2024-11-24T13:47:23,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,609 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,609 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=900, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,609 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043601) roll requested 2024-11-24T13:47:23,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043609 2024-11-24T13:47:23,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043601 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043609 2024-11-24T13:47:23,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,618 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,619 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=901, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,619 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043609) roll requested 2024-11-24T13:47:23,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043619 2024-11-24T13:47:23,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043609 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043619 2024-11-24T13:47:23,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,628 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,628 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=902, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,628 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043619) roll requested 2024-11-24T13:47:23,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043628 2024-11-24T13:47:23,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043619 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043628 2024-11-24T13:47:23,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,638 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,638 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=903, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,638 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043628) roll requested 2024-11-24T13:47:23,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043638 2024-11-24T13:47:23,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043628 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043638 2024-11-24T13:47:23,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,646 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,646 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=904, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,646 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043638) roll requested 2024-11-24T13:47:23,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043646 2024-11-24T13:47:23,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043638 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043646 2024-11-24T13:47:23,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,653 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,654 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=905, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,654 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043646) roll requested 2024-11-24T13:47:23,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043654 2024-11-24T13:47:23,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043646 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043654 2024-11-24T13:47:23,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,662 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,663 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=906, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,663 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043654) roll requested 2024-11-24T13:47:23,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043663 2024-11-24T13:47:23,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043654 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043663 2024-11-24T13:47:23,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,671 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,671 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=907, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,671 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043663) roll requested 2024-11-24T13:47:23,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043671 2024-11-24T13:47:23,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043663 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043671 2024-11-24T13:47:23,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,681 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,681 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=908, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,681 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043671) roll requested 2024-11-24T13:47:23,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043681 2024-11-24T13:47:23,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043671 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043681 2024-11-24T13:47:23,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,689 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,689 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=909, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,689 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043681) roll requested 2024-11-24T13:47:23,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043690 2024-11-24T13:47:23,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043690 2024-11-24T13:47:23,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,699 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,699 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=910, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,699 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043690) roll requested 2024-11-24T13:47:23,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043699 2024-11-24T13:47:23,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043690 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043699 2024-11-24T13:47:23,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,707 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,708 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=911, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,708 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043699) roll requested 2024-11-24T13:47:23,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043708 2024-11-24T13:47:23,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043699 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043708 2024-11-24T13:47:23,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,718 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,718 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=912, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,718 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043708) roll requested 2024-11-24T13:47:23,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043718 2024-11-24T13:47:23,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043708 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043718 2024-11-24T13:47:23,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,726 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,726 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=913, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,726 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043718) roll requested 2024-11-24T13:47:23,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043727 2024-11-24T13:47:23,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043718 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043727 2024-11-24T13:47:23,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,734 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,734 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=914, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,734 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043727) roll requested 2024-11-24T13:47:23,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043735 2024-11-24T13:47:23,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043727 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043735 2024-11-24T13:47:23,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,743 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,743 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=915, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,743 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043735) roll requested 2024-11-24T13:47:23,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043743 2024-11-24T13:47:23,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043735 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043743 2024-11-24T13:47:23,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,751 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,751 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=916, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,751 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043743) roll requested 2024-11-24T13:47:23,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043751 2024-11-24T13:47:23,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043743 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043751 2024-11-24T13:47:23,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,758 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,758 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=917, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,758 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043751) roll requested 2024-11-24T13:47:23,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043758 2024-11-24T13:47:23,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043751 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043758 2024-11-24T13:47:23,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,764 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,765 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=918, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,765 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043758) roll requested 2024-11-24T13:47:23,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043765 2024-11-24T13:47:23,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043758 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043765 2024-11-24T13:47:23,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,772 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,773 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=919, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,773 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043765) roll requested 2024-11-24T13:47:23,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043773 2024-11-24T13:47:23,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043765 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043773 2024-11-24T13:47:23,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,781 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,782 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=920, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,782 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043773) roll requested 2024-11-24T13:47:23,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043782 2024-11-24T13:47:23,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043773 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043782 2024-11-24T13:47:23,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,790 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,791 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=921, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,791 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043782) roll requested 2024-11-24T13:47:23,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043791 2024-11-24T13:47:23,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043782 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043791 2024-11-24T13:47:23,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,799 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,799 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=922, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,799 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043791) roll requested 2024-11-24T13:47:23,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043799 2024-11-24T13:47:23,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043791 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043799 2024-11-24T13:47:23,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,806 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,806 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=923, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,806 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043799) roll requested 2024-11-24T13:47:23,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043807 2024-11-24T13:47:23,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043799 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043807 2024-11-24T13:47:23,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,815 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,816 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=924, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,816 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043807) roll requested 2024-11-24T13:47:23,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043816 2024-11-24T13:47:23,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043807 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043816 2024-11-24T13:47:23,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,828 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,829 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=925, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,829 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043816) roll requested 2024-11-24T13:47:23,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043829 2024-11-24T13:47:23,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043816 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043829 2024-11-24T13:47:23,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,837 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,838 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=926, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,838 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043829) roll requested 2024-11-24T13:47:23,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043838 2024-11-24T13:47:23,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043829 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043838 2024-11-24T13:47:23,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,846 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,847 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=927, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,847 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043838) roll requested 2024-11-24T13:47:23,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043847 2024-11-24T13:47:23,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043838 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043847 2024-11-24T13:47:23,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,856 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,856 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=928, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,856 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043847) roll requested 2024-11-24T13:47:23,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043857 2024-11-24T13:47:23,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043847 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043857 2024-11-24T13:47:23,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,866 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,867 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=929, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,867 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043857) roll requested 2024-11-24T13:47:23,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043867 2024-11-24T13:47:23,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043857 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043867 2024-11-24T13:47:23,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,875 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,875 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=930, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,875 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043867) roll requested 2024-11-24T13:47:23,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043876 2024-11-24T13:47:23,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043867 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043876 2024-11-24T13:47:23,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,881 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,881 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=931, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,882 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043876) roll requested 2024-11-24T13:47:23,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043882 2024-11-24T13:47:23,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043876 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043882 2024-11-24T13:47:23,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,888 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,888 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=932, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,888 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043882) roll requested 2024-11-24T13:47:23,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043889 2024-11-24T13:47:23,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043882 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043889 2024-11-24T13:47:23,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,894 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,895 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=933, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,895 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043889) roll requested 2024-11-24T13:47:23,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043895 2024-11-24T13:47:23,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043889 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043895 2024-11-24T13:47:23,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,901 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,902 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=934, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,902 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043895) roll requested 2024-11-24T13:47:23,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043902 2024-11-24T13:47:23,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043895 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043902 2024-11-24T13:47:23,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,909 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,910 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=935, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,910 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043902) roll requested 2024-11-24T13:47:23,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043910 2024-11-24T13:47:23,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043902 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043910 2024-11-24T13:47:23,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,919 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,920 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=936, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,920 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043910) roll requested 2024-11-24T13:47:23,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043920 2024-11-24T13:47:23,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043910 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043920 2024-11-24T13:47:23,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,929 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,929 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=937, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,929 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043920) roll requested 2024-11-24T13:47:23,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043929 2024-11-24T13:47:23,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043920 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043929 2024-11-24T13:47:23,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,936 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,936 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=938, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,936 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043929) roll requested 2024-11-24T13:47:23,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043936 2024-11-24T13:47:23,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043929 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043936 2024-11-24T13:47:23,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,944 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,944 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=939, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,944 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043936) roll requested 2024-11-24T13:47:23,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043944 2024-11-24T13:47:23,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043936 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043944 2024-11-24T13:47:23,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,952 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=940, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,952 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043944) roll requested 2024-11-24T13:47:23,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043953 2024-11-24T13:47:23,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043944 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043953 2024-11-24T13:47:23,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,960 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,960 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=941, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,960 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043953) roll requested 2024-11-24T13:47:23,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043960 2024-11-24T13:47:23,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043953 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043960 2024-11-24T13:47:23,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,967 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,968 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=942, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,968 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043960) roll requested 2024-11-24T13:47:23,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043968 2024-11-24T13:47:23,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043960 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043968 2024-11-24T13:47:23,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,975 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,975 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=943, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,975 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043968) roll requested 2024-11-24T13:47:23,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043975 2024-11-24T13:47:23,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043968 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043975 2024-11-24T13:47:23,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,983 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,984 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=944, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,984 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043975) roll requested 2024-11-24T13:47:23,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043984 2024-11-24T13:47:23,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:23,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043975 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043984 2024-11-24T13:47:23,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:23,993 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:23,994 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=945, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:23,994 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:23,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043984) roll requested 2024-11-24T13:47:23,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456043994 2024-11-24T13:47:24,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043984 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043994 2024-11-24T13:47:24,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,003 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,003 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=946, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,003 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456043994) roll requested 2024-11-24T13:47:24,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044003 2024-11-24T13:47:24,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456043994 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044003 2024-11-24T13:47:24,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,008 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,008 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=947, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,008 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044003) roll requested 2024-11-24T13:47:24,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044008 2024-11-24T13:47:24,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044003 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044008 2024-11-24T13:47:24,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,013 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,014 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=948, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,014 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044008) roll requested 2024-11-24T13:47:24,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044014 2024-11-24T13:47:24,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044008 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044014 2024-11-24T13:47:24,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,021 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,022 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=949, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,022 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044014) roll requested 2024-11-24T13:47:24,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044022 2024-11-24T13:47:24,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044014 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044022 2024-11-24T13:47:24,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,029 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,029 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=950, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,030 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044022) roll requested 2024-11-24T13:47:24,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044030 2024-11-24T13:47:24,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044022 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044030 2024-11-24T13:47:24,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,036 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,036 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=951, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,037 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044030) roll requested 2024-11-24T13:47:24,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044037 2024-11-24T13:47:24,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044030 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044037 2024-11-24T13:47:24,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,045 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,046 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=952, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,046 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044037) roll requested 2024-11-24T13:47:24,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044046 2024-11-24T13:47:24,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044037 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044046 2024-11-24T13:47:24,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,054 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,054 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=953, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,054 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044046) roll requested 2024-11-24T13:47:24,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044054 2024-11-24T13:47:24,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044046 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044054 2024-11-24T13:47:24,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,062 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,062 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=954, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,062 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044054) roll requested 2024-11-24T13:47:24,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044063 2024-11-24T13:47:24,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044054 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044063 2024-11-24T13:47:24,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,070 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,070 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=955, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,070 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044063) roll requested 2024-11-24T13:47:24,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044070 2024-11-24T13:47:24,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044063 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044070 2024-11-24T13:47:24,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,077 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,077 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=956, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,078 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044070) roll requested 2024-11-24T13:47:24,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044078 2024-11-24T13:47:24,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044070 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044078 2024-11-24T13:47:24,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,086 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,086 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=957, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,086 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044078) roll requested 2024-11-24T13:47:24,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044086 2024-11-24T13:47:24,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044078 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044086 2024-11-24T13:47:24,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,094 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,094 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=958, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,095 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044086) roll requested 2024-11-24T13:47:24,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044095 2024-11-24T13:47:24,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044086 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044095 2024-11-24T13:47:24,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,103 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,103 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=959, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,104 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044095) roll requested 2024-11-24T13:47:24,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044104 2024-11-24T13:47:24,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044095 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044104 2024-11-24T13:47:24,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,112 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,112 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=960, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,112 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044104) roll requested 2024-11-24T13:47:24,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044112 2024-11-24T13:47:24,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044104 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044112 2024-11-24T13:47:24,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,120 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,120 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=961, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,121 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044112) roll requested 2024-11-24T13:47:24,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044121 2024-11-24T13:47:24,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044112 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044121 2024-11-24T13:47:24,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,129 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,129 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=962, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,129 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044121) roll requested 2024-11-24T13:47:24,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044129 2024-11-24T13:47:24,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044121 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044129 2024-11-24T13:47:24,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,138 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,138 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=963, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,139 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044129) roll requested 2024-11-24T13:47:24,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044139 2024-11-24T13:47:24,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044129 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044139 2024-11-24T13:47:24,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,148 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,149 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=964, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,149 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044139) roll requested 2024-11-24T13:47:24,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044149 2024-11-24T13:47:24,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044139 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044149 2024-11-24T13:47:24,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,159 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,159 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=965, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,159 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044149) roll requested 2024-11-24T13:47:24,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044159 2024-11-24T13:47:24,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044149 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044159 2024-11-24T13:47:24,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,169 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,169 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=966, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,169 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044159) roll requested 2024-11-24T13:47:24,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044169 2024-11-24T13:47:24,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044159 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044169 2024-11-24T13:47:24,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,178 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,179 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=967, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,179 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044169) roll requested 2024-11-24T13:47:24,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044179 2024-11-24T13:47:24,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044169 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044179 2024-11-24T13:47:24,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,189 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,189 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=968, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,189 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044179) roll requested 2024-11-24T13:47:24,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044189 2024-11-24T13:47:24,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044179 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044189 2024-11-24T13:47:24,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,198 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,198 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=969, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,198 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044189) roll requested 2024-11-24T13:47:24,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044199 2024-11-24T13:47:24,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044189 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044199 2024-11-24T13:47:24,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,208 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,208 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=970, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,208 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044199) roll requested 2024-11-24T13:47:24,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044208 2024-11-24T13:47:24,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044199 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044208 2024-11-24T13:47:24,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,215 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,216 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=971, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,216 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044208) roll requested 2024-11-24T13:47:24,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044216 2024-11-24T13:47:24,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044208 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044216 2024-11-24T13:47:24,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,223 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,223 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=972, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,223 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044216) roll requested 2024-11-24T13:47:24,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044223 2024-11-24T13:47:24,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044216 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044223 2024-11-24T13:47:24,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,239 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,239 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=973, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,239 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044223) roll requested 2024-11-24T13:47:24,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044239 2024-11-24T13:47:24,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044223 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044239 2024-11-24T13:47:24,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,246 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,247 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=974, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,247 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044239) roll requested 2024-11-24T13:47:24,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044247 2024-11-24T13:47:24,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044239 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044247 2024-11-24T13:47:24,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,252 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,252 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=975, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,252 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044247) roll requested 2024-11-24T13:47:24,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044252 2024-11-24T13:47:24,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044247 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044252 2024-11-24T13:47:24,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,257 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,257 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=976, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,257 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044252) roll requested 2024-11-24T13:47:24,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044257 2024-11-24T13:47:24,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044252 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044257 2024-11-24T13:47:24,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,263 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,263 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=977, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,263 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044257) roll requested 2024-11-24T13:47:24,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044263 2024-11-24T13:47:24,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044257 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044263 2024-11-24T13:47:24,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,268 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,269 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=978, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,269 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044263) roll requested 2024-11-24T13:47:24,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044269 2024-11-24T13:47:24,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044263 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044269 2024-11-24T13:47:24,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,275 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,275 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=979, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,275 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044269) roll requested 2024-11-24T13:47:24,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044275 2024-11-24T13:47:24,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044269 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044275 2024-11-24T13:47:24,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,283 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=980, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,283 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,283 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044275) roll requested 2024-11-24T13:47:24,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044283 2024-11-24T13:47:24,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044275 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044283 2024-11-24T13:47:24,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,291 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,291 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=981, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,291 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044283) roll requested 2024-11-24T13:47:24,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044291 2024-11-24T13:47:24,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044283 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044291 2024-11-24T13:47:24,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,301 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,301 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=982, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,301 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044291) roll requested 2024-11-24T13:47:24,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044301 2024-11-24T13:47:24,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044291 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044301 2024-11-24T13:47:24,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,307 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,307 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=983, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,307 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044301) roll requested 2024-11-24T13:47:24,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044307 2024-11-24T13:47:24,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044301 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044307 2024-11-24T13:47:24,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,312 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,312 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=984, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,312 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044307) roll requested 2024-11-24T13:47:24,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044312 2024-11-24T13:47:24,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044307 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044312 2024-11-24T13:47:24,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,318 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,318 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=985, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,318 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044312) roll requested 2024-11-24T13:47:24,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044318 2024-11-24T13:47:24,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044312 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044318 2024-11-24T13:47:24,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,324 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,325 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=986, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,325 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044318) roll requested 2024-11-24T13:47:24,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044325 2024-11-24T13:47:24,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044318 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044325 2024-11-24T13:47:24,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,332 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,332 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=987, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,332 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044325) roll requested 2024-11-24T13:47:24,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044332 2024-11-24T13:47:24,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044325 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044332 2024-11-24T13:47:24,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,338 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,338 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=988, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,339 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044332) roll requested 2024-11-24T13:47:24,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044339 2024-11-24T13:47:24,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044332 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044339 2024-11-24T13:47:24,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,346 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,346 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=989, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,346 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044339) roll requested 2024-11-24T13:47:24,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044347 2024-11-24T13:47:24,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044339 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044347 2024-11-24T13:47:24,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,354 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,355 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=990, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,355 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044347) roll requested 2024-11-24T13:47:24,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044355 2024-11-24T13:47:24,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044347 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044355 2024-11-24T13:47:24,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,363 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,363 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=991, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,363 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044355) roll requested 2024-11-24T13:47:24,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044364 2024-11-24T13:47:24,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044355 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044364 2024-11-24T13:47:24,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,372 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,373 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=992, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,373 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044364) roll requested 2024-11-24T13:47:24,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044373 2024-11-24T13:47:24,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044364 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044373 2024-11-24T13:47:24,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,382 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,382 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=993, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,382 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044373) roll requested 2024-11-24T13:47:24,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044382 2024-11-24T13:47:24,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044373 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044382 2024-11-24T13:47:24,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,392 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,392 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=994, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,392 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044382) roll requested 2024-11-24T13:47:24,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044393 2024-11-24T13:47:24,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044382 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044393 2024-11-24T13:47:24,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,399 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,400 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=995, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,400 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044393) roll requested 2024-11-24T13:47:24,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044400 2024-11-24T13:47:24,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044393 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044400 2024-11-24T13:47:24,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,411 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,411 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=996, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,411 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044400) roll requested 2024-11-24T13:47:24,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044411 2024-11-24T13:47:24,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044400 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044411 2024-11-24T13:47:24,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,419 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,419 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=997, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,419 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044411) roll requested 2024-11-24T13:47:24,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044419 2024-11-24T13:47:24,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044411 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044419 2024-11-24T13:47:24,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,427 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,427 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=998, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,427 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044419) roll requested 2024-11-24T13:47:24,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044427 2024-11-24T13:47:24,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044419 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044427 2024-11-24T13:47:24,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,435 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,435 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=999, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,435 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044427) roll requested 2024-11-24T13:47:24,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044435 2024-11-24T13:47:24,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044427 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044435 2024-11-24T13:47:24,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,441 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,441 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1000, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,441 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044435) roll requested 2024-11-24T13:47:24,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044441 2024-11-24T13:47:24,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044435 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044441 2024-11-24T13:47:24,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,446 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,446 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1001, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,447 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044441) roll requested 2024-11-24T13:47:24,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044447 2024-11-24T13:47:24,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044441 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044447 2024-11-24T13:47:24,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,451 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,452 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1002, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,452 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044447) roll requested 2024-11-24T13:47:24,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044452 2024-11-24T13:47:24,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044447 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044452 2024-11-24T13:47:24,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,460 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,460 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1003, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,460 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044452) roll requested 2024-11-24T13:47:24,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044460 2024-11-24T13:47:24,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044452 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044460 2024-11-24T13:47:24,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,467 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,467 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1004, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,467 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044460) roll requested 2024-11-24T13:47:24,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044467 2024-11-24T13:47:24,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044460 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044467 2024-11-24T13:47:24,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,474 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,474 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1005, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,474 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044467) roll requested 2024-11-24T13:47:24,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044474 2024-11-24T13:47:24,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044467 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044474 2024-11-24T13:47:24,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,482 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,482 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1006, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,482 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044474) roll requested 2024-11-24T13:47:24,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044482 2024-11-24T13:47:24,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044474 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044482 2024-11-24T13:47:24,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,494 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,495 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1007, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,495 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044482) roll requested 2024-11-24T13:47:24,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044495 2024-11-24T13:47:24,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044482 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044495 2024-11-24T13:47:24,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,503 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,503 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1008, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,503 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044495) roll requested 2024-11-24T13:47:24,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044503 2024-11-24T13:47:24,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044495 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044503 2024-11-24T13:47:24,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,510 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,511 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1009, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,511 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044503) roll requested 2024-11-24T13:47:24,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044511 2024-11-24T13:47:24,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044503 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044511 2024-11-24T13:47:24,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,516 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,516 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1010, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,517 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044511) roll requested 2024-11-24T13:47:24,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044517 2024-11-24T13:47:24,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044511 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044517 2024-11-24T13:47:24,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,522 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,522 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1011, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,522 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044517) roll requested 2024-11-24T13:47:24,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044522 2024-11-24T13:47:24,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044517 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044522 2024-11-24T13:47:24,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,529 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,530 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1012, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,530 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044522) roll requested 2024-11-24T13:47:24,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044530 2024-11-24T13:47:24,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044522 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044530 2024-11-24T13:47:24,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,536 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,536 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1013, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,536 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044530) roll requested 2024-11-24T13:47:24,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044536 2024-11-24T13:47:24,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044530 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044536 2024-11-24T13:47:24,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,542 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,543 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1014, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,543 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044536) roll requested 2024-11-24T13:47:24,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044543 2024-11-24T13:47:24,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044536 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044543 2024-11-24T13:47:24,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,550 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,550 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1015, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,550 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044543) roll requested 2024-11-24T13:47:24,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044550 2024-11-24T13:47:24,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044543 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044550 2024-11-24T13:47:24,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,561 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,562 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1016, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,562 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044550) roll requested 2024-11-24T13:47:24,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044562 2024-11-24T13:47:24,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044550 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044562 2024-11-24T13:47:24,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,575 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,575 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1017, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,575 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044562) roll requested 2024-11-24T13:47:24,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044575 2024-11-24T13:47:24,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044562 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044575 2024-11-24T13:47:24,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,583 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,583 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1018, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,583 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044575) roll requested 2024-11-24T13:47:24,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044583 2024-11-24T13:47:24,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044575 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044583 2024-11-24T13:47:24,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,589 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,589 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1019, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,589 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044583) roll requested 2024-11-24T13:47:24,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044589 2024-11-24T13:47:24,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044583 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044589 2024-11-24T13:47:24,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,597 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,597 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1020, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,598 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044589) roll requested 2024-11-24T13:47:24,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044598 2024-11-24T13:47:24,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044589 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044598 2024-11-24T13:47:24,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,602 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,602 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1021, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,602 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044598) roll requested 2024-11-24T13:47:24,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044603 2024-11-24T13:47:24,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044598 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044603 2024-11-24T13:47:24,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,607 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,607 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1022, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,607 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044603) roll requested 2024-11-24T13:47:24,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044607 2024-11-24T13:47:24,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044603 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044607 2024-11-24T13:47:24,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,614 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,614 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1023, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,614 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044607) roll requested 2024-11-24T13:47:24,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044614 2024-11-24T13:47:24,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044607 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044614 2024-11-24T13:47:24,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,620 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,621 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1024, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,621 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044614) roll requested 2024-11-24T13:47:24,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044621 2024-11-24T13:47:24,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044614 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044621 2024-11-24T13:47:24,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,629 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,629 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1025, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,629 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044621) roll requested 2024-11-24T13:47:24,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044629 2024-11-24T13:47:24,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044621 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044629 2024-11-24T13:47:24,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,635 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,635 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1026, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,635 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044629) roll requested 2024-11-24T13:47:24,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044635 2024-11-24T13:47:24,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044629 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044635 2024-11-24T13:47:24,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,642 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,642 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1027, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,642 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044635) roll requested 2024-11-24T13:47:24,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044642 2024-11-24T13:47:24,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044635 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044642 2024-11-24T13:47:24,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,649 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,650 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1028, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,650 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044642) roll requested 2024-11-24T13:47:24,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044650 2024-11-24T13:47:24,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044642 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044650 2024-11-24T13:47:24,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,657 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,658 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1029, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,658 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044650) roll requested 2024-11-24T13:47:24,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044658 2024-11-24T13:47:24,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044650 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044658 2024-11-24T13:47:24,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,666 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,667 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1030, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,667 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044658) roll requested 2024-11-24T13:47:24,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044667 2024-11-24T13:47:24,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044658 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044667 2024-11-24T13:47:24,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,675 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,675 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1031, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,675 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044667) roll requested 2024-11-24T13:47:24,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044675 2024-11-24T13:47:24,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044667 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044675 2024-11-24T13:47:24,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,682 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,683 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1032, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,683 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044675) roll requested 2024-11-24T13:47:24,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044683 2024-11-24T13:47:24,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044675 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044683 2024-11-24T13:47:24,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,697 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,697 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1033, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,698 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044683) roll requested 2024-11-24T13:47:24,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044698 2024-11-24T13:47:24,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044683 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044698 2024-11-24T13:47:24,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,708 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1034, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,708 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044698) roll requested 2024-11-24T13:47:24,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044708 2024-11-24T13:47:24,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044698 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044708 2024-11-24T13:47:24,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,716 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,716 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1035, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,716 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044708) roll requested 2024-11-24T13:47:24,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044716 2024-11-24T13:47:24,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044708 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044716 2024-11-24T13:47:24,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,721 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,722 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1036, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,722 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044716) roll requested 2024-11-24T13:47:24,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044722 2024-11-24T13:47:24,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044716 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044722 2024-11-24T13:47:24,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,727 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,728 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1037, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,728 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044722) roll requested 2024-11-24T13:47:24,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044728 2024-11-24T13:47:24,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044722 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044728 2024-11-24T13:47:24,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,734 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,734 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1038, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,735 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044728) roll requested 2024-11-24T13:47:24,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044735 2024-11-24T13:47:24,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044728 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044735 2024-11-24T13:47:24,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,741 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,742 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1039, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,742 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044735) roll requested 2024-11-24T13:47:24,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044742 2024-11-24T13:47:24,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044735 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044742 2024-11-24T13:47:24,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,750 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,750 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1040, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,750 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044742) roll requested 2024-11-24T13:47:24,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044750 2024-11-24T13:47:24,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044742 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044750 2024-11-24T13:47:24,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,758 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,758 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1041, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,758 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044750) roll requested 2024-11-24T13:47:24,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044759 2024-11-24T13:47:24,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044750 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044759 2024-11-24T13:47:24,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,766 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,766 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1042, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,766 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044759) roll requested 2024-11-24T13:47:24,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044766 2024-11-24T13:47:24,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044759 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044766 2024-11-24T13:47:24,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,772 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,772 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1043, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,773 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044766) roll requested 2024-11-24T13:47:24,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044773 2024-11-24T13:47:24,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044766 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044773 2024-11-24T13:47:24,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,780 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,780 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1044, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,780 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044773) roll requested 2024-11-24T13:47:24,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044780 2024-11-24T13:47:24,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044773 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044780 2024-11-24T13:47:24,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,788 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,788 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1045, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,788 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044780) roll requested 2024-11-24T13:47:24,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044788 2024-11-24T13:47:24,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044780 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044788 2024-11-24T13:47:24,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,795 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,795 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1046, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,795 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044788) roll requested 2024-11-24T13:47:24,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044795 2024-11-24T13:47:24,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044788 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044795 2024-11-24T13:47:24,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,802 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,803 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1047, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,803 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044795) roll requested 2024-11-24T13:47:24,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044803 2024-11-24T13:47:24,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044795 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044803 2024-11-24T13:47:24,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,810 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,811 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1048, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,811 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044803) roll requested 2024-11-24T13:47:24,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044811 2024-11-24T13:47:24,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044803 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044811 2024-11-24T13:47:24,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,819 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,819 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1049, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,819 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044811) roll requested 2024-11-24T13:47:24,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044819 2024-11-24T13:47:24,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044811 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044819 2024-11-24T13:47:24,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,828 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,828 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1050, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,828 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044819) roll requested 2024-11-24T13:47:24,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044829 2024-11-24T13:47:24,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044819 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044829 2024-11-24T13:47:24,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,839 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,840 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1051, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,840 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044829) roll requested 2024-11-24T13:47:24,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044840 2024-11-24T13:47:24,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044829 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044840 2024-11-24T13:47:24,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,846 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,847 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1052, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,847 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044840) roll requested 2024-11-24T13:47:24,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044847 2024-11-24T13:47:24,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044840 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044847 2024-11-24T13:47:24,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,852 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,852 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1053, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,852 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044847) roll requested 2024-11-24T13:47:24,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044852 2024-11-24T13:47:24,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044847 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044852 2024-11-24T13:47:24,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,859 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,859 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1054, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,859 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044852) roll requested 2024-11-24T13:47:24,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044859 2024-11-24T13:47:24,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044852 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044859 2024-11-24T13:47:24,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,866 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,867 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1055, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,867 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044859) roll requested 2024-11-24T13:47:24,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044867 2024-11-24T13:47:24,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044859 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044867 2024-11-24T13:47:24,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,873 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,874 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1056, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,874 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044867) roll requested 2024-11-24T13:47:24,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044874 2024-11-24T13:47:24,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044867 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044874 2024-11-24T13:47:24,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,882 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,882 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1057, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,882 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044874) roll requested 2024-11-24T13:47:24,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044882 2024-11-24T13:47:24,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044874 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044882 2024-11-24T13:47:24,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,890 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,891 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1058, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,891 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044882) roll requested 2024-11-24T13:47:24,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044891 2024-11-24T13:47:24,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044882 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044891 2024-11-24T13:47:24,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,900 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,901 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1059, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,901 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044891) roll requested 2024-11-24T13:47:24,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044901 2024-11-24T13:47:24,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044891 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044901 2024-11-24T13:47:24,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,910 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,911 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1060, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,911 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044901) roll requested 2024-11-24T13:47:24,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044911 2024-11-24T13:47:24,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044901 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044911 2024-11-24T13:47:24,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,921 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,921 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1061, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,921 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044911) roll requested 2024-11-24T13:47:24,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044922 2024-11-24T13:47:24,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044911 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044922 2024-11-24T13:47:24,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,930 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,931 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1062, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,931 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044922) roll requested 2024-11-24T13:47:24,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044931 2024-11-24T13:47:24,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044922 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044931 2024-11-24T13:47:24,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,941 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,941 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1063, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,942 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044931) roll requested 2024-11-24T13:47:24,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044942 2024-11-24T13:47:24,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044931 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044942 2024-11-24T13:47:24,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,951 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,951 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1064, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,951 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044942) roll requested 2024-11-24T13:47:24,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044951 2024-11-24T13:47:24,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044942 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044951 2024-11-24T13:47:24,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,956 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,956 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1065, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,956 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044951) roll requested 2024-11-24T13:47:24,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044956 2024-11-24T13:47:24,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044951 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044956 2024-11-24T13:47:24,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,961 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,962 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1066, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,962 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044956) roll requested 2024-11-24T13:47:24,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044962 2024-11-24T13:47:24,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044956 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044962 2024-11-24T13:47:24,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,969 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,969 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1067, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,969 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044962) roll requested 2024-11-24T13:47:24,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044969 2024-11-24T13:47:24,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044962 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044969 2024-11-24T13:47:24,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,976 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,977 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1068, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,977 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044969) roll requested 2024-11-24T13:47:24,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044977 2024-11-24T13:47:24,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044969 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044977 2024-11-24T13:47:24,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,984 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,984 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1069, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,984 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044977) roll requested 2024-11-24T13:47:24,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044985 2024-11-24T13:47:24,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:24,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044977 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044985 2024-11-24T13:47:24,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:24,993 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:24,993 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1070, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:24,993 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:24,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044985) roll requested 2024-11-24T13:47:24,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456044993 2024-11-24T13:47:25,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044985 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044993 2024-11-24T13:47:25,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,003 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,003 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1071, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,003 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456044993) roll requested 2024-11-24T13:47:25,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045003 2024-11-24T13:47:25,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456044993 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045003 2024-11-24T13:47:25,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,008 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,009 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1072, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,009 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045003) roll requested 2024-11-24T13:47:25,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045009 2024-11-24T13:47:25,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045003 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045009 2024-11-24T13:47:25,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,014 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,014 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1073, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,014 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045009) roll requested 2024-11-24T13:47:25,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045014 2024-11-24T13:47:25,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045009 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045014 2024-11-24T13:47:25,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,021 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,021 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1074, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,021 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045014) roll requested 2024-11-24T13:47:25,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045021 2024-11-24T13:47:25,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045014 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045021 2024-11-24T13:47:25,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,033 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,034 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1075, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,034 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045021) roll requested 2024-11-24T13:47:25,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045034 2024-11-24T13:47:25,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045021 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045034 2024-11-24T13:47:25,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,040 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,040 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1076, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,040 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045034) roll requested 2024-11-24T13:47:25,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045040 2024-11-24T13:47:25,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045034 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045040 2024-11-24T13:47:25,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,046 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,046 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1077, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,047 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045040) roll requested 2024-11-24T13:47:25,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045047 2024-11-24T13:47:25,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045040 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045047 2024-11-24T13:47:25,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,053 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,054 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1078, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,054 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045047) roll requested 2024-11-24T13:47:25,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045054 2024-11-24T13:47:25,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045047 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045054 2024-11-24T13:47:25,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,063 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,063 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1079, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,064 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045054) roll requested 2024-11-24T13:47:25,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045064 2024-11-24T13:47:25,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045054 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045064 2024-11-24T13:47:25,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,073 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1080, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,073 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,073 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045064) roll requested 2024-11-24T13:47:25,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045073 2024-11-24T13:47:25,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045064 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045073 2024-11-24T13:47:25,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,079 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,080 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1081, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,080 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045073) roll requested 2024-11-24T13:47:25,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045080 2024-11-24T13:47:25,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045073 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045080 2024-11-24T13:47:25,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,096 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,097 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1082, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,097 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045080) roll requested 2024-11-24T13:47:25,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045097 2024-11-24T13:47:25,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045080 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045097 2024-11-24T13:47:25,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,109 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,109 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1083, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,109 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045097) roll requested 2024-11-24T13:47:25,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045109 2024-11-24T13:47:25,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045097 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045109 2024-11-24T13:47:25,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,116 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,116 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1084, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,117 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045109) roll requested 2024-11-24T13:47:25,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045117 2024-11-24T13:47:25,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045109 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045117 2024-11-24T13:47:25,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,122 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,123 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1085, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,123 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045117) roll requested 2024-11-24T13:47:25,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045123 2024-11-24T13:47:25,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045117 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045123 2024-11-24T13:47:25,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,132 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,133 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1086, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,133 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045123) roll requested 2024-11-24T13:47:25,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045133 2024-11-24T13:47:25,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045123 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045133 2024-11-24T13:47:25,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,140 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,140 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1087, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,140 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045133) roll requested 2024-11-24T13:47:25,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045140 2024-11-24T13:47:25,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045133 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045140 2024-11-24T13:47:25,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,145 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,145 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1088, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,145 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045140) roll requested 2024-11-24T13:47:25,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045145 2024-11-24T13:47:25,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045140 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045145 2024-11-24T13:47:25,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,151 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,151 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1089, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,151 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045145) roll requested 2024-11-24T13:47:25,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045151 2024-11-24T13:47:25,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045145 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045151 2024-11-24T13:47:25,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,157 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,157 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1090, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,157 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045151) roll requested 2024-11-24T13:47:25,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045158 2024-11-24T13:47:25,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045151 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045158 2024-11-24T13:47:25,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,165 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,166 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1091, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,166 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045158) roll requested 2024-11-24T13:47:25,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045166 2024-11-24T13:47:25,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045158 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045166 2024-11-24T13:47:25,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,175 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,175 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1092, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,175 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045166) roll requested 2024-11-24T13:47:25,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045175 2024-11-24T13:47:25,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045166 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045175 2024-11-24T13:47:25,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,185 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,186 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1093, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,186 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045175) roll requested 2024-11-24T13:47:25,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045186 2024-11-24T13:47:25,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045175 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045186 2024-11-24T13:47:25,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,195 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,195 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1094, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,196 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045186) roll requested 2024-11-24T13:47:25,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045196 2024-11-24T13:47:25,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045186 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045196 2024-11-24T13:47:25,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,203 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,203 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1095, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,203 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045196) roll requested 2024-11-24T13:47:25,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045203 2024-11-24T13:47:25,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045196 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045203 2024-11-24T13:47:25,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,210 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,210 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1096, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,210 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045203) roll requested 2024-11-24T13:47:25,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045210 2024-11-24T13:47:25,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045203 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045210 2024-11-24T13:47:25,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,217 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,218 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1097, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,218 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045210) roll requested 2024-11-24T13:47:25,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045218 2024-11-24T13:47:25,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045210 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045218 2024-11-24T13:47:25,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,225 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,225 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1098, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,225 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045218) roll requested 2024-11-24T13:47:25,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045225 2024-11-24T13:47:25,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045218 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045225 2024-11-24T13:47:25,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,233 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,233 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1099, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,234 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045225) roll requested 2024-11-24T13:47:25,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045234 2024-11-24T13:47:25,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045225 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045234 2024-11-24T13:47:25,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,241 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,242 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1100, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,242 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045234) roll requested 2024-11-24T13:47:25,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045242 2024-11-24T13:47:25,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045234 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045242 2024-11-24T13:47:25,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,250 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,250 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1101, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,251 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045242) roll requested 2024-11-24T13:47:25,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045251 2024-11-24T13:47:25,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045242 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045251 2024-11-24T13:47:25,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,258 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,258 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1102, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,258 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045251) roll requested 2024-11-24T13:47:25,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045259 2024-11-24T13:47:25,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045251 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045259 2024-11-24T13:47:25,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,266 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,266 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1103, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,267 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045259) roll requested 2024-11-24T13:47:25,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045267 2024-11-24T13:47:25,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045259 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045267 2024-11-24T13:47:25,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,275 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,276 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1104, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,276 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045267) roll requested 2024-11-24T13:47:25,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045276 2024-11-24T13:47:25,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045267 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045276 2024-11-24T13:47:25,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,285 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,286 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1105, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,286 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045276) roll requested 2024-11-24T13:47:25,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045286 2024-11-24T13:47:25,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045276 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045286 2024-11-24T13:47:25,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,295 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,295 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1106, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,296 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045286) roll requested 2024-11-24T13:47:25,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045296 2024-11-24T13:47:25,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045286 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045296 2024-11-24T13:47:25,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,302 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,302 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1107, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,302 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045296) roll requested 2024-11-24T13:47:25,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045302 2024-11-24T13:47:25,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045296 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045302 2024-11-24T13:47:25,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,308 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,309 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1108, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,309 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045302) roll requested 2024-11-24T13:47:25,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045309 2024-11-24T13:47:25,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045302 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045309 2024-11-24T13:47:25,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,316 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,317 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1109, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,317 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045309) roll requested 2024-11-24T13:47:25,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045317 2024-11-24T13:47:25,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045309 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045317 2024-11-24T13:47:25,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,326 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,326 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1110, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,326 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045317) roll requested 2024-11-24T13:47:25,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045326 2024-11-24T13:47:25,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045317 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045326 2024-11-24T13:47:25,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,336 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,336 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1111, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,336 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045326) roll requested 2024-11-24T13:47:25,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045336 2024-11-24T13:47:25,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045326 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045336 2024-11-24T13:47:25,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,344 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,344 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1112, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,344 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045336) roll requested 2024-11-24T13:47:25,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045344 2024-11-24T13:47:25,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045336 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045344 2024-11-24T13:47:25,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,353 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,353 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1113, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,353 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045344) roll requested 2024-11-24T13:47:25,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045353 2024-11-24T13:47:25,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045344 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045353 2024-11-24T13:47:25,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,358 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,358 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1114, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,359 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045353) roll requested 2024-11-24T13:47:25,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045359 2024-11-24T13:47:25,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045353 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045359 2024-11-24T13:47:25,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,363 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,364 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1115, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,364 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045359) roll requested 2024-11-24T13:47:25,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045364 2024-11-24T13:47:25,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045359 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045364 2024-11-24T13:47:25,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,369 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,370 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1116, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,370 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045364) roll requested 2024-11-24T13:47:25,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045370 2024-11-24T13:47:25,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045364 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045370 2024-11-24T13:47:25,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,377 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,377 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1117, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,377 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045370) roll requested 2024-11-24T13:47:25,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045377 2024-11-24T13:47:25,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045370 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045377 2024-11-24T13:47:25,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,384 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,385 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1118, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,385 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045377) roll requested 2024-11-24T13:47:25,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045385 2024-11-24T13:47:25,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045377 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045385 2024-11-24T13:47:25,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,394 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,395 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1119, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,395 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045385) roll requested 2024-11-24T13:47:25,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045395 2024-11-24T13:47:25,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045385 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045395 2024-11-24T13:47:25,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,403 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,403 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1120, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,403 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045395) roll requested 2024-11-24T13:47:25,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045403 2024-11-24T13:47:25,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045395 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045403 2024-11-24T13:47:25,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,408 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1121, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,408 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045403) roll requested 2024-11-24T13:47:25,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045409 2024-11-24T13:47:25,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045403 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045409 2024-11-24T13:47:25,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,414 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,414 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1122, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,414 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045409) roll requested 2024-11-24T13:47:25,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045414 2024-11-24T13:47:25,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045409 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045414 2024-11-24T13:47:25,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,420 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,421 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1123, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,421 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045414) roll requested 2024-11-24T13:47:25,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045421 2024-11-24T13:47:25,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045414 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045421 2024-11-24T13:47:25,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,429 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,429 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1124, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,429 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045421) roll requested 2024-11-24T13:47:25,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045429 2024-11-24T13:47:25,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045421 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045429 2024-11-24T13:47:25,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,437 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,437 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1125, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,437 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045429) roll requested 2024-11-24T13:47:25,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045437 2024-11-24T13:47:25,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045429 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045437 2024-11-24T13:47:25,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,446 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,446 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1126, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,446 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045437) roll requested 2024-11-24T13:47:25,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045446 2024-11-24T13:47:25,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045437 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045446 2024-11-24T13:47:25,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,455 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,455 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1127, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,455 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045446) roll requested 2024-11-24T13:47:25,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045455 2024-11-24T13:47:25,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045446 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045455 2024-11-24T13:47:25,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,461 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,461 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1128, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,461 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045455) roll requested 2024-11-24T13:47:25,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045461 2024-11-24T13:47:25,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045455 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045461 2024-11-24T13:47:25,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,467 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,467 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1129, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,467 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045461) roll requested 2024-11-24T13:47:25,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045467 2024-11-24T13:47:25,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045461 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045467 2024-11-24T13:47:25,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,472 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,472 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1130, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,472 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045467) roll requested 2024-11-24T13:47:25,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045472 2024-11-24T13:47:25,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045467 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045472 2024-11-24T13:47:25,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,477 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,478 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1131, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,478 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045472) roll requested 2024-11-24T13:47:25,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045478 2024-11-24T13:47:25,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045472 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045478 2024-11-24T13:47:25,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,484 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,484 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1132, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,484 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045478) roll requested 2024-11-24T13:47:25,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045484 2024-11-24T13:47:25,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045478 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045484 2024-11-24T13:47:25,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,490 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,491 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1133, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,491 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045484) roll requested 2024-11-24T13:47:25,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045491 2024-11-24T13:47:25,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045484 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045491 2024-11-24T13:47:25,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,498 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,499 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1134, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,499 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045491) roll requested 2024-11-24T13:47:25,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045499 2024-11-24T13:47:25,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045491 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045499 2024-11-24T13:47:25,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,508 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,508 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1135, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,508 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045499) roll requested 2024-11-24T13:47:25,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045508 2024-11-24T13:47:25,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045499 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045508 2024-11-24T13:47:25,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,513 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,513 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1136, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,513 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045508) roll requested 2024-11-24T13:47:25,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045513 2024-11-24T13:47:25,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045508 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045513 2024-11-24T13:47:25,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,523 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,524 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1137, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,524 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045513) roll requested 2024-11-24T13:47:25,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045524 2024-11-24T13:47:25,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045513 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045524 2024-11-24T13:47:25,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,532 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,532 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1138, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,532 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045524) roll requested 2024-11-24T13:47:25,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045532 2024-11-24T13:47:25,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045524 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045532 2024-11-24T13:47:25,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,543 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,544 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1139, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,544 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045532) roll requested 2024-11-24T13:47:25,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045544 2024-11-24T13:47:25,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045532 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045544 2024-11-24T13:47:25,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,554 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,554 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1140, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,554 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045544) roll requested 2024-11-24T13:47:25,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045554 2024-11-24T13:47:25,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045544 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045554 2024-11-24T13:47:25,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,560 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,561 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1141, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,561 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045554) roll requested 2024-11-24T13:47:25,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045561 2024-11-24T13:47:25,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045554 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045561 2024-11-24T13:47:25,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,565 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,566 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1142, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,566 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045561) roll requested 2024-11-24T13:47:25,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045566 2024-11-24T13:47:25,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045561 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045566 2024-11-24T13:47:25,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,570 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,571 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1143, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,571 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045566) roll requested 2024-11-24T13:47:25,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045571 2024-11-24T13:47:25,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045566 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045571 2024-11-24T13:47:25,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,576 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,577 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1144, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,577 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045571) roll requested 2024-11-24T13:47:25,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045577 2024-11-24T13:47:25,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045571 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045577 2024-11-24T13:47:25,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,582 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,583 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1145, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,583 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045577) roll requested 2024-11-24T13:47:25,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045583 2024-11-24T13:47:25,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045577 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045583 2024-11-24T13:47:25,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,589 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,590 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1146, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,590 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045583) roll requested 2024-11-24T13:47:25,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045590 2024-11-24T13:47:25,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045583 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045590 2024-11-24T13:47:25,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,597 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,597 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1147, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,597 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045590) roll requested 2024-11-24T13:47:25,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045598 2024-11-24T13:47:25,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045590 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045598 2024-11-24T13:47:25,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,606 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,606 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1148, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,606 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045598) roll requested 2024-11-24T13:47:25,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045606 2024-11-24T13:47:25,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045598 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045606 2024-11-24T13:47:25,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,615 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,616 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1149, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,616 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045606) roll requested 2024-11-24T13:47:25,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045616 2024-11-24T13:47:25,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045606 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045616 2024-11-24T13:47:25,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,624 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,625 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1150, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,625 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045616) roll requested 2024-11-24T13:47:25,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045625 2024-11-24T13:47:25,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045616 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045625 2024-11-24T13:47:25,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,634 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,635 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1151, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,635 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045625) roll requested 2024-11-24T13:47:25,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045635 2024-11-24T13:47:25,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045625 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045635 2024-11-24T13:47:25,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,642 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,642 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1152, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,642 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045635) roll requested 2024-11-24T13:47:25,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045642 2024-11-24T13:47:25,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045635 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045642 2024-11-24T13:47:25,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,650 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,651 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1153, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,651 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045642) roll requested 2024-11-24T13:47:25,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045651 2024-11-24T13:47:25,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045642 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045651 2024-11-24T13:47:25,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,657 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,658 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1154, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,658 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045651) roll requested 2024-11-24T13:47:25,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045658 2024-11-24T13:47:25,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045651 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045658 2024-11-24T13:47:25,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,666 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,666 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1155, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,667 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045658) roll requested 2024-11-24T13:47:25,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045667 2024-11-24T13:47:25,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045658 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045667 2024-11-24T13:47:25,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,674 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,674 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1156, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,674 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045667) roll requested 2024-11-24T13:47:25,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045674 2024-11-24T13:47:25,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045667 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045674 2024-11-24T13:47:25,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,681 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,682 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1157, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,682 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045674) roll requested 2024-11-24T13:47:25,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045682 2024-11-24T13:47:25,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045674 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045682 2024-11-24T13:47:25,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,690 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,691 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1158, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,691 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045682) roll requested 2024-11-24T13:47:25,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045691 2024-11-24T13:47:25,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045682 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045691 2024-11-24T13:47:25,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,698 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,699 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1159, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,699 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045691) roll requested 2024-11-24T13:47:25,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045699 2024-11-24T13:47:25,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045691 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045699 2024-11-24T13:47:25,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,707 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,708 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1160, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,708 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045699) roll requested 2024-11-24T13:47:25,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045708 2024-11-24T13:47:25,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045699 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045708 2024-11-24T13:47:25,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,718 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,719 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1161, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,719 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045708) roll requested 2024-11-24T13:47:25,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045719 2024-11-24T13:47:25,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045708 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045719 2024-11-24T13:47:25,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,731 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,732 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1162, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,732 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045719) roll requested 2024-11-24T13:47:25,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045732 2024-11-24T13:47:25,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045719 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045732 2024-11-24T13:47:25,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,737 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,738 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1163, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,738 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045732) roll requested 2024-11-24T13:47:25,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045738 2024-11-24T13:47:25,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045732 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045738 2024-11-24T13:47:25,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,742 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,743 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1164, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,743 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045738) roll requested 2024-11-24T13:47:25,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045743 2024-11-24T13:47:25,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045738 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045743 2024-11-24T13:47:25,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,748 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,748 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1165, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,748 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045743) roll requested 2024-11-24T13:47:25,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045748 2024-11-24T13:47:25,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045743 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045748 2024-11-24T13:47:25,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,754 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,755 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1166, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,755 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045748) roll requested 2024-11-24T13:47:25,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045755 2024-11-24T13:47:25,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045748 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045755 2024-11-24T13:47:25,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,762 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,762 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1167, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,762 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045755) roll requested 2024-11-24T13:47:25,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045762 2024-11-24T13:47:25,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045755 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045762 2024-11-24T13:47:25,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,770 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,771 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1168, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,771 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045762) roll requested 2024-11-24T13:47:25,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045771 2024-11-24T13:47:25,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045762 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045771 2024-11-24T13:47:25,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,778 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,779 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1169, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,779 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045771) roll requested 2024-11-24T13:47:25,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045779 2024-11-24T13:47:25,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045771 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045779 2024-11-24T13:47:25,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,787 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,788 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1170, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,788 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045779) roll requested 2024-11-24T13:47:25,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045788 2024-11-24T13:47:25,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045779 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045788 2024-11-24T13:47:25,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,797 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,798 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1171, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,798 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045788) roll requested 2024-11-24T13:47:25,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045798 2024-11-24T13:47:25,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045788 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045798 2024-11-24T13:47:25,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,805 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,806 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1172, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,806 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045798) roll requested 2024-11-24T13:47:25,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045806 2024-11-24T13:47:25,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045798 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045806 2024-11-24T13:47:25,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,812 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,813 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1173, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,813 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045806) roll requested 2024-11-24T13:47:25,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045813 2024-11-24T13:47:25,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045806 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045813 2024-11-24T13:47:25,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,821 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,821 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1174, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,821 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045813) roll requested 2024-11-24T13:47:25,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045821 2024-11-24T13:47:25,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045813 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045821 2024-11-24T13:47:25,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,830 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,830 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1175, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,830 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045821) roll requested 2024-11-24T13:47:25,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045830 2024-11-24T13:47:25,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045821 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045830 2024-11-24T13:47:25,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,838 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,839 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1176, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,839 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045830) roll requested 2024-11-24T13:47:25,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045839 2024-11-24T13:47:25,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045830 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045839 2024-11-24T13:47:25,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,849 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,850 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1177, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,850 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045839) roll requested 2024-11-24T13:47:25,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045850 2024-11-24T13:47:25,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045839 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045850 2024-11-24T13:47:25,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,858 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,859 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1178, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,859 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045850) roll requested 2024-11-24T13:47:25,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045859 2024-11-24T13:47:25,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045850 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045859 2024-11-24T13:47:25,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,866 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,867 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1179, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,867 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045859) roll requested 2024-11-24T13:47:25,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045867 2024-11-24T13:47:25,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045859 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045867 2024-11-24T13:47:25,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,875 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,875 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1180, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,875 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045867) roll requested 2024-11-24T13:47:25,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045875 2024-11-24T13:47:25,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045867 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045875 2024-11-24T13:47:25,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,884 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,885 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1181, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,885 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045875) roll requested 2024-11-24T13:47:25,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045885 2024-11-24T13:47:25,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045875 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045885 2024-11-24T13:47:25,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,901 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,902 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1182, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,902 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045885) roll requested 2024-11-24T13:47:25,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045902 2024-11-24T13:47:25,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045885 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045902 2024-11-24T13:47:25,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,911 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,911 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1183, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,911 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045902) roll requested 2024-11-24T13:47:25,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045911 2024-11-24T13:47:25,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045902 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045911 2024-11-24T13:47:25,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,919 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,919 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1184, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,919 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045911) roll requested 2024-11-24T13:47:25,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045919 2024-11-24T13:47:25,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045911 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045919 2024-11-24T13:47:25,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,925 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,925 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1185, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,925 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045919) roll requested 2024-11-24T13:47:25,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045925 2024-11-24T13:47:25,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045919 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045925 2024-11-24T13:47:25,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,931 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,931 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1186, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,931 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045925) roll requested 2024-11-24T13:47:25,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045931 2024-11-24T13:47:25,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045925 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045931 2024-11-24T13:47:25,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,940 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,940 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1187, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,940 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045931) roll requested 2024-11-24T13:47:25,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045940 2024-11-24T13:47:25,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045931 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045940 2024-11-24T13:47:25,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,945 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,946 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1188, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,946 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045940) roll requested 2024-11-24T13:47:25,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045946 2024-11-24T13:47:25,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045940 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045946 2024-11-24T13:47:25,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,954 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,954 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1189, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,954 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045946) roll requested 2024-11-24T13:47:25,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045954 2024-11-24T13:47:25,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045946 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045954 2024-11-24T13:47:25,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,960 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,960 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1190, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,960 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045954) roll requested 2024-11-24T13:47:25,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045960 2024-11-24T13:47:25,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045954 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045960 2024-11-24T13:47:25,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,967 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,967 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1191, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,967 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045960) roll requested 2024-11-24T13:47:25,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045967 2024-11-24T13:47:25,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045960 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045967 2024-11-24T13:47:25,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,972 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,973 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1192, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,973 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045967) roll requested 2024-11-24T13:47:25,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045973 2024-11-24T13:47:25,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045967 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045973 2024-11-24T13:47:25,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,978 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,979 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1193, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,979 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045973) roll requested 2024-11-24T13:47:25,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045979 2024-11-24T13:47:25,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045973 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045979 2024-11-24T13:47:25,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,984 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,985 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1194, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,985 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045979) roll requested 2024-11-24T13:47:25,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045985 2024-11-24T13:47:25,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045979 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045985 2024-11-24T13:47:25,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,991 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,991 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1195, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,991 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045985) roll requested 2024-11-24T13:47:25,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045991 2024-11-24T13:47:25,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:25,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045985 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045991 2024-11-24T13:47:25,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:25,996 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:25,996 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1196, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:25,996 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:25,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045991) roll requested 2024-11-24T13:47:25,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456045997 2024-11-24T13:47:26,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045991 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045997 2024-11-24T13:47:26,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,002 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,002 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1197, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,002 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456045997) roll requested 2024-11-24T13:47:26,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046002 2024-11-24T13:47:26,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456045997 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046002 2024-11-24T13:47:26,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,007 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,007 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1198, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,008 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046002) roll requested 2024-11-24T13:47:26,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046008 2024-11-24T13:47:26,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046002 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046008 2024-11-24T13:47:26,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,016 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,016 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1199, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,016 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046008) roll requested 2024-11-24T13:47:26,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046016 2024-11-24T13:47:26,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046008 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046016 2024-11-24T13:47:26,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,024 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,025 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1200, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,025 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046016) roll requested 2024-11-24T13:47:26,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046025 2024-11-24T13:47:26,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046016 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046025 2024-11-24T13:47:26,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,032 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,033 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1201, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,033 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046025) roll requested 2024-11-24T13:47:26,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046033 2024-11-24T13:47:26,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046025 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046033 2024-11-24T13:47:26,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,042 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,042 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1202, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,042 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046033) roll requested 2024-11-24T13:47:26,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046042 2024-11-24T13:47:26,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046033 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046042 2024-11-24T13:47:26,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,049 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,049 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1203, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,049 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046042) roll requested 2024-11-24T13:47:26,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046049 2024-11-24T13:47:26,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046042 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046049 2024-11-24T13:47:26,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,054 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,055 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1204, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,055 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046049) roll requested 2024-11-24T13:47:26,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046055 2024-11-24T13:47:26,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046049 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046055 2024-11-24T13:47:26,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,062 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,062 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1205, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,063 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046055) roll requested 2024-11-24T13:47:26,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046063 2024-11-24T13:47:26,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046055 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046063 2024-11-24T13:47:26,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,070 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,071 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1206, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,071 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046063) roll requested 2024-11-24T13:47:26,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046071 2024-11-24T13:47:26,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046063 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046071 2024-11-24T13:47:26,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,077 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,077 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1207, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,077 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046071) roll requested 2024-11-24T13:47:26,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046077 2024-11-24T13:47:26,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046071 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046077 2024-11-24T13:47:26,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,084 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,084 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1208, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,084 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046077) roll requested 2024-11-24T13:47:26,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046084 2024-11-24T13:47:26,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046077 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046084 2024-11-24T13:47:26,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,092 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,093 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1209, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,093 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046084) roll requested 2024-11-24T13:47:26,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046093 2024-11-24T13:47:26,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046084 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046093 2024-11-24T13:47:26,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,102 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,103 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1210, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,103 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046093) roll requested 2024-11-24T13:47:26,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046103 2024-11-24T13:47:26,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046093 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046103 2024-11-24T13:47:26,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,114 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,114 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1211, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,114 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046103) roll requested 2024-11-24T13:47:26,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046114 2024-11-24T13:47:26,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046103 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046114 2024-11-24T13:47:26,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,123 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,124 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1212, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,124 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046114) roll requested 2024-11-24T13:47:26,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046124 2024-11-24T13:47:26,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046114 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046124 2024-11-24T13:47:26,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,132 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,132 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1213, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,132 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046124) roll requested 2024-11-24T13:47:26,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046132 2024-11-24T13:47:26,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046124 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046132 2024-11-24T13:47:26,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,141 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,142 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1214, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,142 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046132) roll requested 2024-11-24T13:47:26,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046142 2024-11-24T13:47:26,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046132 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046142 2024-11-24T13:47:26,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,154 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,155 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1215, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,155 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046142) roll requested 2024-11-24T13:47:26,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046155 2024-11-24T13:47:26,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046142 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046155 2024-11-24T13:47:26,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,160 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,160 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1216, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,160 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046155) roll requested 2024-11-24T13:47:26,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046160 2024-11-24T13:47:26,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046155 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046160 2024-11-24T13:47:26,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,165 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,166 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1217, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,166 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046160) roll requested 2024-11-24T13:47:26,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046166 2024-11-24T13:47:26,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046160 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046166 2024-11-24T13:47:26,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,173 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,174 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1218, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,174 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046166) roll requested 2024-11-24T13:47:26,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046174 2024-11-24T13:47:26,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046166 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046174 2024-11-24T13:47:26,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,186 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,186 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1219, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,186 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046174) roll requested 2024-11-24T13:47:26,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046186 2024-11-24T13:47:26,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046174 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046186 2024-11-24T13:47:26,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,191 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,192 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1220, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,192 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046186) roll requested 2024-11-24T13:47:26,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046192 2024-11-24T13:47:26,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046186 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046192 2024-11-24T13:47:26,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,196 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,197 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1221, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,197 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046192) roll requested 2024-11-24T13:47:26,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046197 2024-11-24T13:47:26,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046192 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046197 2024-11-24T13:47:26,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,203 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,203 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1222, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,203 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046197) roll requested 2024-11-24T13:47:26,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046203 2024-11-24T13:47:26,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046197 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046203 2024-11-24T13:47:26,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,210 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,210 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1223, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,210 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046203) roll requested 2024-11-24T13:47:26,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046210 2024-11-24T13:47:26,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046203 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046210 2024-11-24T13:47:26,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,218 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,218 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1224, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,218 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046210) roll requested 2024-11-24T13:47:26,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046219 2024-11-24T13:47:26,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046210 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046219 2024-11-24T13:47:26,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,228 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1225, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,229 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046219) roll requested 2024-11-24T13:47:26,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046229 2024-11-24T13:47:26,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046219 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046229 2024-11-24T13:47:26,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,238 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,238 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1226, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,238 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046229) roll requested 2024-11-24T13:47:26,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046238 2024-11-24T13:47:26,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046229 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046238 2024-11-24T13:47:26,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,245 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,245 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1227, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,245 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046238) roll requested 2024-11-24T13:47:26,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046245 2024-11-24T13:47:26,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046238 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046245 2024-11-24T13:47:26,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,253 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,253 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1228, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,253 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046245) roll requested 2024-11-24T13:47:26,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046253 2024-11-24T13:47:26,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046245 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046253 2024-11-24T13:47:26,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,261 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,262 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1229, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,262 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046253) roll requested 2024-11-24T13:47:26,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046262 2024-11-24T13:47:26,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046253 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046262 2024-11-24T13:47:26,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,270 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,270 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1230, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,271 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046262) roll requested 2024-11-24T13:47:26,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046271 2024-11-24T13:47:26,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046262 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046271 2024-11-24T13:47:26,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,278 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,279 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1231, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,279 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046271) roll requested 2024-11-24T13:47:26,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046279 2024-11-24T13:47:26,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046271 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046279 2024-11-24T13:47:26,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,286 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,287 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1232, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,287 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046279) roll requested 2024-11-24T13:47:26,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046287 2024-11-24T13:47:26,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046279 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046287 2024-11-24T13:47:26,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,294 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,294 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1233, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,294 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046287) roll requested 2024-11-24T13:47:26,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046295 2024-11-24T13:47:26,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046287 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046295 2024-11-24T13:47:26,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,303 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,304 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1234, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,304 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046295) roll requested 2024-11-24T13:47:26,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046304 2024-11-24T13:47:26,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046295 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046304 2024-11-24T13:47:26,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,312 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,313 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1235, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,313 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046304) roll requested 2024-11-24T13:47:26,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046313 2024-11-24T13:47:26,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046304 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046313 2024-11-24T13:47:26,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,327 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,327 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1236, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,327 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046313) roll requested 2024-11-24T13:47:26,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046327 2024-11-24T13:47:26,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046313 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046327 2024-11-24T13:47:26,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,332 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,333 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1237, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,333 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046327) roll requested 2024-11-24T13:47:26,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046333 2024-11-24T13:47:26,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046327 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046333 2024-11-24T13:47:26,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,338 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,338 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1238, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,338 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046333) roll requested 2024-11-24T13:47:26,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046338 2024-11-24T13:47:26,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046333 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046338 2024-11-24T13:47:26,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,344 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,345 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1239, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,345 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046338) roll requested 2024-11-24T13:47:26,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046345 2024-11-24T13:47:26,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046338 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046345 2024-11-24T13:47:26,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,351 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,351 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1240, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,352 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046345) roll requested 2024-11-24T13:47:26,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046352 2024-11-24T13:47:26,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046345 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046352 2024-11-24T13:47:26,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,359 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,360 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1241, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,360 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046352) roll requested 2024-11-24T13:47:26,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046360 2024-11-24T13:47:26,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046352 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046360 2024-11-24T13:47:26,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,369 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,369 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1242, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,370 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046360) roll requested 2024-11-24T13:47:26,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046370 2024-11-24T13:47:26,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046360 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046370 2024-11-24T13:47:26,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,376 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,376 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1243, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,377 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046370) roll requested 2024-11-24T13:47:26,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046377 2024-11-24T13:47:26,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046370 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046377 2024-11-24T13:47:26,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,385 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,386 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1244, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,386 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046377) roll requested 2024-11-24T13:47:26,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046386 2024-11-24T13:47:26,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046377 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046386 2024-11-24T13:47:26,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,409 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1245, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,409 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046386) roll requested 2024-11-24T13:47:26,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046409 2024-11-24T13:47:26,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046386 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046409 2024-11-24T13:47:26,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,417 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,418 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1246, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,418 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046409) roll requested 2024-11-24T13:47:26,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046418 2024-11-24T13:47:26,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046409 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046418 2024-11-24T13:47:26,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,428 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,429 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1247, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,429 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046418) roll requested 2024-11-24T13:47:26,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046429 2024-11-24T13:47:26,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046418 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046429 2024-11-24T13:47:26,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,438 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,438 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1248, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,438 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046429) roll requested 2024-11-24T13:47:26,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046438 2024-11-24T13:47:26,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046429 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046438 2024-11-24T13:47:26,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,445 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,446 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1249, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,446 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046438) roll requested 2024-11-24T13:47:26,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046446 2024-11-24T13:47:26,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046438 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046446 2024-11-24T13:47:26,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,451 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,451 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1250, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,451 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046446) roll requested 2024-11-24T13:47:26,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046451 2024-11-24T13:47:26,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046446 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046451 2024-11-24T13:47:26,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,457 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,457 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1251, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,457 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046451) roll requested 2024-11-24T13:47:26,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046457 2024-11-24T13:47:26,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046451 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046457 2024-11-24T13:47:26,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,463 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,463 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1252, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,463 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046457) roll requested 2024-11-24T13:47:26,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046463 2024-11-24T13:47:26,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046457 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046463 2024-11-24T13:47:26,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,470 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,470 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1253, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,470 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046463) roll requested 2024-11-24T13:47:26,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046470 2024-11-24T13:47:26,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046463 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046470 2024-11-24T13:47:26,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,479 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,479 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1254, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,479 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046470) roll requested 2024-11-24T13:47:26,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046479 2024-11-24T13:47:26,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046470 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046479 2024-11-24T13:47:26,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,485 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,485 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1255, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,485 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046479) roll requested 2024-11-24T13:47:26,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046485 2024-11-24T13:47:26,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046479 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046485 2024-11-24T13:47:26,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,490 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,491 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1256, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,491 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046485) roll requested 2024-11-24T13:47:26,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046491 2024-11-24T13:47:26,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046491 2024-11-24T13:47:26,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,497 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,497 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1257, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,497 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046491) roll requested 2024-11-24T13:47:26,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046497 2024-11-24T13:47:26,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046491 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046497 2024-11-24T13:47:26,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,504 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,505 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1258, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,505 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046497) roll requested 2024-11-24T13:47:26,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046505 2024-11-24T13:47:26,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046497 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046505 2024-11-24T13:47:26,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,512 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,512 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1259, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,512 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046505) roll requested 2024-11-24T13:47:26,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046513 2024-11-24T13:47:26,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046505 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046513 2024-11-24T13:47:26,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,522 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,522 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1260, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,522 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046513) roll requested 2024-11-24T13:47:26,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046523 2024-11-24T13:47:26,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046513 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046523 2024-11-24T13:47:26,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,531 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,531 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1261, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,531 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046523) roll requested 2024-11-24T13:47:26,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046531 2024-11-24T13:47:26,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046523 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046531 2024-11-24T13:47:26,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,539 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,539 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1262, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,539 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046531) roll requested 2024-11-24T13:47:26,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046539 2024-11-24T13:47:26,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046531 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046539 2024-11-24T13:47:26,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,547 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,548 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1263, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,548 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046539) roll requested 2024-11-24T13:47:26,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046548 2024-11-24T13:47:26,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046539 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046548 2024-11-24T13:47:26,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,556 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,557 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1264, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,557 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046548) roll requested 2024-11-24T13:47:26,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046557 2024-11-24T13:47:26,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046548 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046557 2024-11-24T13:47:26,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,565 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,566 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1265, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,566 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046557) roll requested 2024-11-24T13:47:26,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046566 2024-11-24T13:47:26,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046557 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046566 2024-11-24T13:47:26,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,575 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,576 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1266, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,576 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046566) roll requested 2024-11-24T13:47:26,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046576 2024-11-24T13:47:26,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046566 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046576 2024-11-24T13:47:26,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,586 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,587 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1267, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,587 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046576) roll requested 2024-11-24T13:47:26,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046587 2024-11-24T13:47:26,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046576 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046587 2024-11-24T13:47:26,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,596 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,597 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1268, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,597 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046587) roll requested 2024-11-24T13:47:26,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046597 2024-11-24T13:47:26,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046587 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046597 2024-11-24T13:47:26,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,606 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,606 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1269, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,607 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046597) roll requested 2024-11-24T13:47:26,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046607 2024-11-24T13:47:26,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046597 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046607 2024-11-24T13:47:26,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,616 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,616 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1270, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,617 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046607) roll requested 2024-11-24T13:47:26,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046617 2024-11-24T13:47:26,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046607 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046617 2024-11-24T13:47:26,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,623 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,624 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1271, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,624 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046617) roll requested 2024-11-24T13:47:26,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046624 2024-11-24T13:47:26,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046617 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046624 2024-11-24T13:47:26,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,630 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,631 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1272, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,631 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046624) roll requested 2024-11-24T13:47:26,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046631 2024-11-24T13:47:26,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046624 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046631 2024-11-24T13:47:26,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,638 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,639 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1273, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,639 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046631) roll requested 2024-11-24T13:47:26,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046639 2024-11-24T13:47:26,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046631 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046639 2024-11-24T13:47:26,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,646 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,647 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1274, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,647 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046639) roll requested 2024-11-24T13:47:26,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046647 2024-11-24T13:47:26,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046639 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046647 2024-11-24T13:47:26,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,654 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,654 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1275, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,654 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046647) roll requested 2024-11-24T13:47:26,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046654 2024-11-24T13:47:26,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046647 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046654 2024-11-24T13:47:26,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,662 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,663 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1276, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,663 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046654) roll requested 2024-11-24T13:47:26,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046663 2024-11-24T13:47:26,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046654 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046663 2024-11-24T13:47:26,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,670 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,671 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1277, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,671 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046663) roll requested 2024-11-24T13:47:26,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046671 2024-11-24T13:47:26,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046663 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046671 2024-11-24T13:47:26,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,679 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,679 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1278, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,679 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046671) roll requested 2024-11-24T13:47:26,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046679 2024-11-24T13:47:26,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046671 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046679 2024-11-24T13:47:26,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,686 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,686 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1279, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,687 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046679) roll requested 2024-11-24T13:47:26,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046687 2024-11-24T13:47:26,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046679 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046687 2024-11-24T13:47:26,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,694 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T13:47:26,695 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1280, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,695 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732456046687) roll requested 2024-11-24T13:47:26,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732456046695 2024-11-24T13:47:26,701 ERROR [Time-limited test {}] wal.AbstractFSWAL(939): Failed log archiving for the log /a/b, java.io.IOException: throw archival exception at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.archiveLogFile(TestFailedAppendAndSync.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.archive(AbstractFSWAL.java:934) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.archive(TestFailedAppendAndSync.java:143) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync.testLockupAroundBadAssignSync(TestFailedAppendAndSync.java:280) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:47:26,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046687 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732456046695 2024-11-24T13:47:26,704 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f95a3003458fa47569d2022d5cc40612, disabling compactions & flushes 2024-11-24T13:47:26,704 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testLockupAroundBadAssignSync,,1732456016378.f95a3003458fa47569d2022d5cc40612. 2024-11-24T13:47:26,704 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testLockupAroundBadAssignSync,,1732456016378.f95a3003458fa47569d2022d5cc40612. 2024-11-24T13:47:26,705 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testLockupAroundBadAssignSync,,1732456016378.f95a3003458fa47569d2022d5cc40612. after waiting 1 ms 2024-11-24T13:47:26,705 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testLockupAroundBadAssignSync,,1732456016378.f95a3003458fa47569d2022d5cc40612. 2024-11-24T13:47:26,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T13:47:26,714 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1281, max=55; forcing (partial) flush of 1 region(s): f95a3003458fa47569d2022d5cc40612[MyCF] 2024-11-24T13:47:26,714 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f95a3003458fa47569d2022d5cc40612, because it is not online on us 2024-11-24T13:47:26,714 INFO [LogRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T13:47:26,714 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 83 in region testLockupAroundBadAssignSync,,1732456016378.f95a3003458fa47569d2022d5cc40612. 2024-11-24T13:47:26,714 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testLockupAroundBadAssignSync,,1732456016378.f95a3003458fa47569d2022d5cc40612. 2024-11-24T13:47:26,714 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f95a3003458fa47569d2022d5cc40612: Waiting for close lock at 1732456046704Disabling compacts and flushes for region at 1732456046704Disabling writes for close at 1732456046705 (+1 ms)Writing region close event to WAL at 1732456046714 (+9 ms)Closed at 1732456046714 2024-11-24T13:47:26,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:26,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T13:47:27,036 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: DodgyFSLog wal:(num 1732456046695) 2024-11-24T13:47:27,037 INFO [Time-limited test {}] regionserver.TestFailedAppendAndSync(105): Cleaning test directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da12a867-8d36-bfa8-69f3-4603c96ebabf 2024-11-24T13:47:27,637 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.TestFailedAppendAndSync#testLockupAroundBadAssignSync Thread=22 (was 11) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: process reaper java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=287 (was 287), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=183 (was 153) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8120 (was 9171)